Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion apps/check/read/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ fn on_path(tool: &str) -> bool {
}

fn open(video: &str) -> R<(Box<dyn VideoDecoder>, raea_io_media::Info)> {
let d = raea_app_media::backend()?.open_video(Path::new(video))?;
let d = raea_app_media::open_video(Path::new(video))?;
let info = d.info().clone();
Ok((d, info))
}
Expand Down
6 changes: 3 additions & 3 deletions apps/media/src/change.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ pub fn video_span(path: &Path, info: &Info) -> f64 {

/// How much the picture changes: the mean absolute luma difference between samples `fps` apart (signalstats YDIF).
pub fn motion(path: &Path, from: Option<f64>, to: Option<f64>, fps: f64) -> R<(Value, Vec<(f64, f64)>)> {
let mut d = backend()?.open_video(path)?;
let mut d = open_video(path)?;
let ts = times(video_span(path, d.info()), from, to, fps);
let mut prev: Option<Vec<u8>> = None;
let mut out = Vec::new();
Expand All @@ -48,7 +48,7 @@ pub fn motion(path: &Path, from: Option<f64>, to: Option<f64>, fps: f64) -> R<(V
/// Shot changes: ffmpeg's scene score (select's get_scene_score: the mean absolute luma difference from the frame before,
/// against the change in it since the frame before that, over 100) above `threshold`, frame by frame.
pub fn cuts(path: &Path, threshold: f64, from: Option<f64>, to: Option<f64>) -> R<(Value, Vec<(usize, f64, f64)>)> {
let mut d = backend()?.open_video(path)?;
let mut d = open_video(path)?;
let info = d.info().clone();
// every frame the file holds, at its own time (a variable-rate take has no fixed step)
let ts = frames_of(path, &info, from, to);
Expand Down Expand Up @@ -86,7 +86,7 @@ pub fn frames_of(path: &Path, info: &Info, from: Option<f64>, to: Option<f64>) -
/// over every plane, as a fraction of full scale); a stretch at or under `noise` lasting `min` seconds or more is a
/// freeze, reported as (start, Some((duration, end))) when it ends, or (start, None) when the file ends frozen.
pub fn freezes(path: &Path, noise: f64, min: f64) -> R<Vec<(f64, Option<(f64, f64)>)>> {
let mut d = backend()?.open_video(path)?;
let mut d = open_video(path)?;
let info = d.info().clone();
let mut reference: Option<(f64, Vec<u8>, Vec<u8>, Vec<u8>)> = None;
let mut frozen_now = false;
Expand Down
15 changes: 15 additions & 0 deletions apps/media/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,21 @@ fn soft() -> R<Box<dyn Backend>> {
raea_io_media::backend("soft")
}

/// A video's decoder from the first backend that reads it (raea_io_media::preferred: RAEA_MEDIA's, the platform's, the
/// portable one, then FFmpeg), so a video the platform's decoder can't read is read through FFmpeg: off a Mac, H.264
/// with B-frames, HEVC and VP9 (on Windows raea fetches its own FFmpeg the first time). Each backend's refusal is kept
/// for the error.
pub fn open_video(path: &Path) -> R<Box<dyn raea_io_media::VideoDecoder>> {
let mut why = Vec::new();
for b in raea_io_media::preferred() {
match b.open_video(path) {
Ok(d) => return Ok(d),
Err(e) => why.push(format!("{}: {e}", b.name())),
}
}
Err(format!("{}: no backend reads it ({})", path.display(), why.join("; ")))
}

const IMAGE_EXTS: &[&str] = &["png", "jpg", "jpeg", "webp", "gif", "bmp", "tif", "tiff"];
const SWITCHES: &[&str] = &["mono"];

Expand Down
2 changes: 1 addition & 1 deletion apps/media/src/picture.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ pub(crate) fn save(img: &image::RgbaImage, out: &Path) -> R<()> {

/// The frame showing at `t` (the last frame when `t` is past the end).
pub(crate) fn frame(path: &Path, t: f64) -> R<image::RgbaImage> {
let mut d = backend()?.open_video(path)?;
let mut d = open_video(path)?;
let dur = d.info().duration;
let t = t.clamp(0.0, (dur - 0.5 / d.info().fps.max(1.0)).max(0.0));
to_image(d.frame_at(t)?)
Expand Down
8 changes: 6 additions & 2 deletions apps/media/src/probe.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ pub fn probe_of(path: &Path) -> R<Probe> {
return Ok(Probe { kind: "image", video: None, audio: None, image: Some((w, h)), duration: 0.0, bytes, track: None });
}
let track = video_track(path);
let video = backend()?.open_video(path).ok().map(|d| d.info().clone()).filter(|i| i.width > 0 && i.frames > 0).map(|mut i| {
let opened = open_video(path);
let why = opened.as_ref().err().cloned();
let video = opened.ok().map(|d| d.info().clone()).filter(|i| i.width > 0 && i.frames > 0).map(|mut i| {
// the file's own timing over what the decoder counted
if let Some(t) = &track {
i.duration = t.duration;
Expand All @@ -38,7 +40,9 @@ pub fn probe_of(path: &Path) -> R<Probe> {
});
let audio = soft()?.decode_audio(path).ok().filter(|a| a.len() > 0).map(|a| (a.rate, a.channels.len(), a.duration()));
if video.is_none() && audio.is_none() {
return Err(format!("{}: no picture or sound raea can read", path.display()));
// what each backend said about the picture: a missing FFmpeg, a codec none of them reads
let why = why.map(|w| format!(" ({})", w.split_once(": no backend reads it (").map_or(w.as_str(), |(_, r)| r.trim_end_matches(')')).to_string())).unwrap_or_default();
return Err(format!("{}: no picture or sound raea can read{why}", path.display()));
}
// the container's own duration when it says one (ffprobe's format duration), else the longest stream
let duration = container_duration(path).unwrap_or_else(|| video.as_ref().map(|v| v.duration).unwrap_or(0.0).max(audio.map(|a| a.2).unwrap_or(0.0)));
Expand Down
2 changes: 1 addition & 1 deletion docs/settings.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ command asks for it (a few minutes, once), or ahead of time with `raea setup-aud
| `RAEA_SAO` | Where Stable Audio Open Small lives (default `~/.cache/raea-sao`, about 2 GB, Core ML): foley and sound effects |
| `RAEA_AUDIO_AUTOSETUP=0` | Don't install on first use; a missing model's error names the command (`tools/sao/setup.sh` on a Mac or Linux, `uv run --no-project --python 3.11 python -I tools\sao\setup.py` on Windows) |
| `RAEA_SAO_CUDA` | SAO off a Mac: `1` installs ONNX Runtime with CUDA, `0` the CPU build; unset, an NVIDIA GPU (`nvidia-smi`) decides |
| `RAEA_TOOLS_CACHE` | Where raea keeps the uv it fetches when there's none on the PATH (default `~/.cache/raea-tools`; uv 0.12.24, checked against its SHA-256); every local model's setup runs through it |
| `RAEA_TOOLS_CACHE` | Where raea keeps the tools it fetches when they're not on the PATH (default `~/.cache/raea-tools`, each checked against its SHA-256): uv 0.12.24, which every local model's setup runs through; and on Windows FFmpeg 9.0.2 (about 210 MB), fetched the first time a video raea's own decoders can't read is opened (a phone's or a camera's: H.264 with B-frames, HEVC, VP9) |

From the command line, `raea sao "PROMPT" -o hit.wav` and `raea tokensynth "TIMBRE" --notes C3,C4 -o
notes/` make sounds through the same cache a composition's render uses, and `raea sounds` searches open sample
Expand Down
2 changes: 1 addition & 1 deletion io/assets/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ pub use splat::{ensure_splat, splat_dir, splat_params};
mod local;
use local::*;
mod setup;
pub use setup::{run_setup, setup_how, uv, venv_python};
pub use setup::{ffmpeg_tool, run_setup, setup_how, uv, venv_python};
#[cfg(test)]
mod tests;

Expand Down
74 changes: 66 additions & 8 deletions io/assets/src/setup.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
//! A local model's install on any OS, with nothing from Homebrew or the system's package manager: its
//! tools/<model>/setup.py, run by uv (which makes the venv and brings its own Python), and the venv's Python run after.
//! uv is the one on the PATH, else raea's own: a pinned release fetched once into ~/.cache/raea-tools/uv and checked
//! against its SHA-256 (where tools/ensure.sh puts the same uv on a Mac, so either finds the other's). Until 0.1.8 each setup was a `sh` script needing uv already installed (`brew install uv`), so Windows
//! had no way to install any local model, and a Mac without Homebrew none either.
//! against its SHA-256 (where tools/ensure.sh puts the same uv on a Mac, so either finds the other's). Until 0.1.8 each
//! setup was a `sh` script needing uv already installed (`brew install uv`), so Windows had no way to install any local
//! model, and a Mac without Homebrew none either. FFmpeg, on Windows (ffmpeg_tool), is fetched the same way the first
//! time a video needs it.

use super::*;

Expand All @@ -28,6 +30,54 @@ fn on_path(name: &str) -> Option<PathBuf> {
std::env::split_paths(&std::env::var_os("PATH")?).map(|d| d.join(&exe)).find(|p| p.is_file())
}

/// The FFmpeg raea fetches on Windows when none is installed: GyanD's essentials build (the one winget's
/// Gyan.FFmpeg.Essentials installs), kept on GitHub for each release; its SHA-256 is GitHub's digest for the asset
const FFMPEG: &str = "9.0.2";
const FFMPEG_SUM: &str = "60f467265b1e312373dbcd92200c2618a74850f98d3d078e94296bb3fa2047ba";

/// `ffmpeg` or `ffprobe` as raea runs it: the one on the PATH; else raea's own in $RAEA_TOOLS_CACHE (else
/// ~/.cache/raea-tools)/ffmpeg/bin. On Windows, where FFmpeg is rarely installed, raea's own is fetched there once
/// (114 MB; ffmpeg.exe, ffprobe.exe and its LICENSE kept, about 210 MB) the first time a video raea's own decoders
/// can't read needs it, when `fetch`: H.264 with B-frames, HEVC, VP9 (a phone's or a camera's). Elsewhere FFmpeg is
/// the system's: a Mac reads those on VideoToolbox, and Linux has FFmpeg in its package manager.
pub fn ffmpeg_tool(name: &str, fetch: bool) -> Result<PathBuf, String> {
if let Some(p) = on_path(name) {
return Ok(p);
}
let dir = raea_io_home::env_path("RAEA_TOOLS_CACHE", ".cache/raea-tools").join("ffmpeg");
let bin = dir.join("bin").join(exe(name));
if bin.is_file() {
return Ok(bin);
}
if !cfg!(windows) {
return Err(format!("{name} isn't installed: raea reads this video through FFmpeg (apt install ffmpeg, dnf install ffmpeg or brew install ffmpeg)"));
}
if !fetch {
return Err(format!("{name} isn't installed yet (raea fetches its own the first time a video needs it)"));
}
// one fetch at a time in a process (a render opens its videos on several threads); another process's is a
// separate .part folder, and whichever finishes first is moved in
static FETCHING: std::sync::Mutex<()> = std::sync::Mutex::new(());
let _one = FETCHING.lock().unwrap_or_else(|e| e.into_inner());
if bin.is_file() {
return Ok(bin);
}
raea_io_home::offline(&format!("fetching FFmpeg {FFMPEG} to read a video raea's own decoders can't"))?;
eprintln!("raea: this video needs FFmpeg, which isn't installed; fetching FFmpeg {FFMPEG} (114 MB, once) into {}", dir.display());
let name_zip = format!("ffmpeg-{FFMPEG}-essentials_build.zip");
let url = format!("https://github.com/GyanD/codexffmpeg/releases/download/{FFMPEG}/{name_zip}");
let top = format!("ffmpeg-{FFMPEG}-essentials_build");
let members = [format!("{top}/bin/ffmpeg.exe"), format!("{top}/bin/ffprobe.exe"), format!("{top}/LICENSE")];
let part = dir.with_file_name(format!("ffmpeg.part{}", std::process::id()));
let fetched = fetch_archive(&part, &url, &name_zip, FFMPEG_SUM, true, &members.each_ref().map(String::as_str));
if fetched.is_ok() && !bin.is_file() {
let _ = std::fs::rename(&part, &dir);
}
let _ = std::fs::remove_dir_all(&part);
fetched?;
if bin.is_file() { Ok(bin) } else { Err(format!("couldn't put FFmpeg in {}", dir.display())) }
}

/// The Python of the venv in an install folder: .venv/bin/python, or .venv\Scripts\python.exe on Windows
pub fn venv_python(dir: &Path) -> PathBuf {
if cfg!(windows) { dir.join(".venv").join("Scripts").join("python.exe") } else { dir.join(".venv").join("bin").join("python") }
Expand Down Expand Up @@ -61,7 +111,8 @@ pub fn uv() -> Result<PathBuf, String> {
eprintln!("raea: fetching uv {UV} ({name}) into {}", dir.display());
// unpacked beside it first and moved in whole, so a stopped fetch never leaves a uv that looks installed
let part = dir.with_file_name(format!("uv.part{}", std::process::id()));
let fetched = fetch_uv(&part, &name, sum, os != "windows");
let url = format!("https://github.com/astral-sh/uv/releases/download/{UV}/{name}");
let fetched = fetch_archive(&part, &url, &name, sum, os != "windows", &[]);
if fetched.is_ok() && !bin.is_file() {
let _ = std::fs::rename(&part, &dir);
}
Expand All @@ -70,13 +121,19 @@ pub fn uv() -> Result<PathBuf, String> {
if bin.is_file() { Ok(bin) } else { Err(format!("couldn't put uv in {}", dir.display())) }
}

/// uv's release archive `name` downloaded into `part` with curl, checked against `sum` and unpacked there by tar
/// (Windows' own curl.exe and tar.exe, which reads a zip: the .tar.gz holds uv-<target>/uv, the zip uv.exe at its top)
fn fetch_uv(part: &Path, name: &str, sum: &str, strip: bool) -> Result<(), String> {
/// A release archive `name` from `url` downloaded into `part` with curl (resumed across its tries: a dropped connection
/// costs only what wasn't fetched), checked against `sum` and unpacked there by tar: `members` alone when given, its top
/// folder stripped when `strip`. Windows' own curl.exe and tar.exe do it there (bsdtar reads a zip).
fn fetch_archive(part: &Path, url: &str, name: &str, sum: &str, strip: bool, members: &[&str]) -> Result<(), String> {
use std::io::IsTerminal;
std::fs::create_dir_all(part).map_err(|e| format!("{}: {e}", part.display()))?;
let archive = part.join(name);
let url = format!("https://github.com/astral-sh/uv/releases/download/{UV}/{name}");
let got = std::process::Command::new(exe("curl")).args(["-fsSL", "--retry", "3", "-o"]).arg(&archive).arg(&url).status();
let quiet = if std::io::stderr().is_terminal() { "--progress-bar" } else { "-sS" };
let got = std::process::Command::new(exe("curl"))
.args(["-fL", quiet, "-C", "-", "--retry", "5", "--retry-all-errors", "--connect-timeout", "30", "-o"])
.arg(&archive)
.arg(url)
.status();
if !got.is_ok_and(|s| s.success()) {
return Err(format!("couldn't download {url} with curl"));
}
Expand All @@ -89,6 +146,7 @@ fn fetch_uv(part: &Path, name: &str, sum: &str, strip: bool) -> Result<(), Strin
if strip {
tar.arg("--strip-components=1");
}
tar.args(members);
if !tar.status().is_ok_and(|s| s.success()) {
return Err(format!("couldn't unpack {name}"));
}
Expand Down
2 changes: 2 additions & 0 deletions io/ffmpeg/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,6 @@ description = "raea's baseline media backend: the ffmpeg command line"

[dependencies]
raea-io-codec = { path = "../codec" }
# raea's own FFmpeg on Windows, fetched the first time a video needs it (ffmpeg_tool)
raea-io-assets = { path = "../assets" }
serde_json = "1"
28 changes: 21 additions & 7 deletions io/ffmpeg/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,20 @@ use raea_io_codec::{

pub struct Ffmpeg;

/// The ffmpeg or ffprobe to run: the PATH's, else raea's own (raea_io_assets::ffmpeg_tool). Opening a video (`fetch`)
/// fetches raea's own on Windows the first time one needs it; sound, encoding and asking what FFmpeg can do (says)
/// use FFmpeg when it's there and never fetch it (a silent clip's sound is tried on every backend).
fn cli(name: &str, fetch: bool) -> R<Command> {
raea_io_assets::ffmpeg_tool(name, fetch).map(Command::new)
}

/// A file named as a video: only these fetch FFmpeg when it isn't there (an audio file or an image never does)
fn named_video(path: &Path) -> bool {
const VIDEO: &[&str] = &["mp4", "mov", "m4v", "mkv", "webm", "avi", "mts", "m2ts", "ts", "3gp", "wmv", "flv", "mpg", "mpeg", "mxf", "ogv"];
let ext = path.extension().map(|e| e.to_string_lossy().to_ascii_lowercase()).unwrap_or_default();
VIDEO.iter().any(|v| *v == ext)
}

/// Every decoded frame, once: -fps_mode from FFmpeg 5.1 on, -vsync before it (FFmpeg 8 took -vsync out, and the decoder
/// read nothing: #113).
pub fn passthrough() -> [&'static str; 2] {
Expand All @@ -28,14 +42,14 @@ pub fn passthrough() -> [&'static str; 2] {

/// What `ffmpeg -hide_banner OPTION` prints (nothing without ffmpeg).
fn says(option: &str) -> String {
let out = Command::new("ffmpeg").args(["-hide_banner", option]).output().map(|o| o.stdout).unwrap_or_default();
let out = cli("ffmpeg", false).ok().and_then(|mut c| c.args(["-hide_banner", option]).output().ok()).map(|o| o.stdout).unwrap_or_default();
String::from_utf8_lossy(&out).into_owned()
}

/// The facts, the presentation times, whether the colour matrix is untagged, and the decoder a VP8 or VP9 stream with
/// alpha needs (libvpx: ffmpeg's own VP9 decoder drops the alpha WebM keeps beside the picture, #113).
fn probe(path: &Path) -> R<(Info, Vec<f64>, bool, Option<&'static str>)> {
let out = Command::new("ffprobe")
let out = cli("ffprobe", named_video(path))?
.args([
"-v",
"error",
Expand Down Expand Up @@ -125,7 +139,7 @@ impl Decoder {
let _ = c.wait();
}
let start = self.pts[at] - self.pts[0];
let mut cmd = Command::new("ffmpeg");
let mut cmd = cli("ffmpeg", false)?;
cmd.args(["-v", "error"]).args(self.vpx.map(|c| ["-c:v", c]).into_iter().flatten()).args(["-ss", &format!("{start:.6}"), "-i"]).arg(&self.path);
if self.bt709 {
cmd.args(["-vf", "scale=in_color_matrix=bt709:in_range=tv"]);
Expand Down Expand Up @@ -220,7 +234,7 @@ impl VideoEncoder for Encoder {
}
}
std::fs::write(&raw, bytes).map_err(|e| e.to_string())?;
let st = Command::new("ffmpeg")
let st = cli("ffmpeg", false)?
.args(["-v", "error", "-y", "-i"])
.arg(&self.video)
.args([
Expand Down Expand Up @@ -289,7 +303,7 @@ impl Backend for Ffmpeg {
}))
}
fn decode_audio(&self, path: &Path) -> R<Audio> {
let ch = Command::new("ffprobe")
let ch = cli("ffprobe", false)?
.args([
"-v",
"error",
Expand All @@ -307,7 +321,7 @@ impl Backend for Ffmpeg {
.trim()
.parse()
.unwrap_or(1);
let out = Command::new("ffmpeg")
let out = cli("ffmpeg", false)?
.args(["-v", "error", "-i"])
.arg(path)
.args(["-vn", "-f", "f32le", "-ar", "48000", "-"])
Expand All @@ -329,7 +343,7 @@ impl Backend for Ffmpeg {
}
fn encoder(&self, out: &Path, o: &EncodeOpts) -> R<Box<dyn VideoEncoder>> {
let video = out.with_extension("video.mp4");
let child = Command::new("ffmpeg")
let child = cli("ffmpeg", false)?
.args([
"-v",
"error",
Expand Down
Loading