From 4d44519fc16a51eafb53593c1aa8e2bc5aefa63c Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Wed, 18 Nov 2026 16:05:07 -0800 Subject: [PATCH] Local models install and run on a Mac, Windows or Linux with nothing installed first Each local model's setup is now tools//setup.py (standard library only, on tools/local_setup.py), run by uv on any OS as raea runs it: uv on the PATH, else raea's own uv 0.12.24 fetched once into ~/.cache/raea-tools/uv (where tools/ensure.sh puts it) and checked against its SHA-256. The setup.sh beside each is a wrapper for a Mac or Linux. Until now each setup was a sh script needing uv already installed (brew install uv), so Windows had no way to install any local model. - SAO: Core ML on a Mac, ONNX Runtime elsewhere (CUDA where nvidia-smi finds a GPU; RAEA_SAO_CUDA=0|1); its WAV is written by Python's wave module, not an installed ffmpeg (same samples) - matchmove, splat: footage read through PyAV, whose wheels carry FFmpeg's libraries (no ffmpeg/ffprobe needed); splat fetches Brush v0.3.0's release build for macOS arm64, Linux x86_64 or Windows x86_64, each SHA-256 checked - goo: Taichi on Metal on a Mac, else its best GPU (CUDA, Vulkan) or the CPU; the backend is reported as run - TokenSynth: its code as GitHub's archive at a pinned commit (no git); coremltools only on a Mac - the engine runs .venv\Scripts\python.exe on Windows (venv_python), Brush's brush_app.exe - tools/release.sh ships tools/local_setup.py --- Cargo.lock | 1 + apps/tools/src/sound.rs | 4 +- docs/manual/reference/cli/setup-audio.md | 44 +++---- docs/settings.md | 4 +- docs/sound.feature | 5 +- io/assets/src/lib.rs | 27 ++-- io/assets/src/local.rs | 4 +- io/assets/src/setup.rs | 119 ++++++++++++++++++ io/assets/src/splat.rs | 20 +-- systems/bake/Cargo.toml | 2 + systems/bake/src/goo.rs | 10 +- tools/goo/run.py | 7 +- tools/goo/setup.py | 32 +++++ tools/goo/setup.sh | 49 ++------ tools/local_setup.py | 138 ++++++++++++++++++++ tools/matchmove/run.py | 59 ++++++--- tools/matchmove/setup.py | 20 +++ tools/matchmove/setup.sh | 43 ++----- tools/release.sh | 4 +- tools/sao/run.py | 15 ++- tools/sao/setup.py | 102 +++++++++++++++ tools/sao/setup.sh | 152 ++--------------------- tools/splat/run.py | 49 ++++++-- tools/splat/setup.py | 61 +++++++++ tools/splat/setup.sh | 61 ++------- tools/tokensynth/convert.py | 2 +- tools/tokensynth/run.py | 2 +- tools/tokensynth/setup.py | 50 ++++++++ tools/tokensynth/setup.sh | 75 ++--------- 29 files changed, 752 insertions(+), 409 deletions(-) create mode 100644 io/assets/src/setup.rs create mode 100644 tools/goo/setup.py create mode 100644 tools/local_setup.py create mode 100644 tools/matchmove/setup.py create mode 100644 tools/sao/setup.py create mode 100644 tools/splat/setup.py create mode 100644 tools/tokensynth/setup.py diff --git a/Cargo.lock b/Cargo.lock index 7ea62b58..250077f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15698,6 +15698,7 @@ name = "raea-sys-bake" version = "0.1.0" dependencies = [ "image", + "raea-io-assets", "raea-io-home", "rapier3d", "serde_json", diff --git a/apps/tools/src/sound.rs b/apps/tools/src/sound.rs index 4b4268e4..360dd93c 100644 --- a/apps/tools/src/sound.rs +++ b/apps/tools/src/sound.rs @@ -253,11 +253,11 @@ pub fn setup_audio(args: &[String]) -> R { } let now = raea_io_assets::installed(m, &dir); missing += !now as usize; - let what = if m == "sao" { "foley (Stable Audio Open Small, Core ML)" } else { "music: an instrument's notes (TokenSynth)" }; + let what = if m == "sao" { "foley (Stable Audio Open Small: Core ML on a Mac, else ONNX)" } else { "music: an instrument's notes (TokenSynth)" }; let state = match (was, now) { (true, _) => "installed".to_string(), (false, true) => "installed now".to_string(), - (false, false) => format!("not installed: {} installs it", raea_io_assets::setup_script(m).display()), + (false, false) => format!("not installed: `raea setup-audio` or {} installs it", raea_io_assets::setup_how(&raea_io_assets::setup_script(m))), }; lines.push(format!(" {m:<10} {state}; {what}\n {} ({})", dir.display(), if now { raea_io_home::disk_use(&dir, false) } else { "-".into() })); } diff --git a/docs/manual/reference/cli/setup-audio.md b/docs/manual/reference/cli/setup-audio.md index b74fe143..ab6f8899 100644 --- a/docs/manual/reference/cli/setup-audio.md +++ b/docs/manual/reference/cli/setup-audio.md @@ -14,34 +14,34 @@ raea setup-audio --check raea setup-audio tokensynth ``` -Each model's script (`share/raea/tools/sao/setup.sh` and `share/raea/tools/tokensynth/setup.sh` beside an installed -raea) installs it into `~/.cache/raea-sao` or `~/.cache/raea-tokensynth` (`RAEA_SAO` and `RAEA_TOKENSYNTH` move them): - -- **SAO, about 2 GB:** Stable Audio Open Small's Core ML models from a Hugging Face mirror at a pinned revision, each - file checked against its SHA-256, t5-base's tokenizer, a small Python environment, and a T5 encoder converted once. +Each model's setup (`share/raea/tools/sao/setup.py` and `share/raea/tools/tokensynth/setup.py` beside an installed +raea, each with a `setup.sh` that runs it on a Mac or Linux) installs it into `~/.cache/raea-sao` or +`~/.cache/raea-tokensynth` (`RAEA_SAO` and `RAEA_TOKENSYNTH` move them): + +- **SAO, about 2 GB:** on a Mac, Stable Audio Open Small's Core ML models from a Hugging Face mirror at a pinned + revision, each file checked against its SHA-256, t5-base's tokenizer, a small Python environment, and a T5 encoder + converted once. On Windows and Linux, the same weights as three ONNX graphs (each checked against its SHA-256) on + ONNX Runtime: with CUDA where `nvidia-smi` finds an NVIDIA GPU (its CUDA libraries add a few GB; `RAEA_SAO_CUDA=0` + keeps the CPU build), else on the CPU. Its licence is Stability AI's Community License ([stability.ai/community-license-agreement](https://stability.ai/community-license-agreement)), not an open one. As between you and Stability, you own the sounds it makes, as far as the law allows; they may not be used to create or improve another generative AI model, and Stability's acceptable use policy applies to them. What else the licence asks, of raea and of a commercial user, raea hasn't written down yet: read it before an SAO sound goes into a commercial piece. -- **TokenSynth, about 6 GB:** its code from GitHub, a Python environment with torch, and its weights, DAC and CLAP - from Hugging Face, then a converted fp16 runtime. MIT. - -Both need a network connection and [uv](https://docs.astral.sh/uv/) on the PATH; TokenSynth also needs `git`. On a -Mac without them, `share/raea/tools/ensure.sh uv` and `ensure.sh git` install raea's pinned builds (uv 0.12.24 and -git 2.56.0, each download checked against its SHA-256) and print where; put those folders on the PATH: - -```sh -B=$(dirname "$(realpath "$(command -v raea)")"); T=$B/../share/raea/tools; [ -d "$T" ] || T=$B/../../tools # an install, or a checkout -PATH="$(dirname "$(bash "$T/ensure.sh" uv)"):$(dirname "$(bash "$T/ensure.sh" git)"):$PATH" raea setup-audio -``` - -When a setup fails, the error names its script: run that with `sh` to see why. Exit status 127 is uv not found; 1 -with `xcrun: error: invalid active developer path` is macOS's stand-in git on a Mac without the Command Line Tools -(use the git `ensure.sh` installs); 75 is another setup running in the same folder. `RAEA_OFFLINE` refuses the -download. `RAEA_AUDIO_AUTOSETUP=0` stops a render from installing a model itself, and stops `raea setup-audio` and -`ensure.sh` too: with it set, run the model's script yourself (`sh share/raea/tools/sao/setup.sh`). +- **TokenSynth, about 6 GB:** its code from GitHub at a pinned commit (an archive: no git), a Python environment + with torch, and its weights, DAC and CLAP from Hugging Face, then a converted fp16 runtime (and, on a Mac, Core ML + models for `--precision coreml`). MIT. + +Both need only a network connection, on a Mac, Windows or Linux: no Homebrew, git or FFmpeg. raea runs each setup +through [uv](https://docs.astral.sh/uv/), the one on the PATH or else its own (uv 0.12.24, fetched once into +`~/.cache/raea-tools/uv` and checked against its SHA-256; `RAEA_TOOLS_CACHE` moves it), and uv brings the Python. + +When a setup fails, the error names the command: run it yourself to see why (`sh share/raea/tools/sao/setup.sh` on +a Mac or Linux; on Windows `uv run --no-project --python 3.11 python -I share\raea\tools\sao\setup.py`, with uv +from the PATH or `%USERPROFILE%\.cache\raea-tools\uv`). Exit status 75 is another setup running in the same folder. +`RAEA_OFFLINE` refuses the download. `RAEA_AUDIO_AUTOSETUP=0` stops a render from installing a model itself, and +stops `raea setup-audio` and `ensure.sh` too: with it set, run the model's setup yourself. diff --git a/docs/settings.md b/docs/settings.md index 8636885a..8087cb90 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -29,7 +29,9 @@ command asks for it (a few minutes, once), or ahead of time with `raea setup-aud |---|---| | `RAEA_TOKENSYNTH` | Where TokenSynth lives (default `~/.cache/raea-tokensynth`, about 6 GB): an instrument described in words, one note at a time, for a Tone.js `Sampler` | | `RAEA_SAO` | Where Stable Audio Open Small lives (default `~/.cache/raea-sao`, about 2 GB, Core ML): foley and sound effects | -| `RAEA_AUDIO_AUTOSETUP=0` | Don't install on first use; a missing model's error names the command (`tools/sao/setup.sh`, `tools/tokensynth/setup.sh`) | +| `RAEA_AUDIO_AUTOSETUP=0` | Don't install on first use; a missing model's error names the command (`tools/sao/setup.sh` on a Mac or Linux, `uv run --no-project --python 3.11 python -I tools\sao\setup.py` on Windows) | +| `RAEA_SAO_CUDA` | SAO off a Mac: `1` installs ONNX Runtime with CUDA, `0` the CPU build; unset, an NVIDIA GPU (`nvidia-smi`) decides | +| `RAEA_TOOLS_CACHE` | Where raea keeps the uv it fetches when there's none on the PATH (default `~/.cache/raea-tools`; uv 0.12.24, checked against its SHA-256); every local model's setup runs through it | From the command line, `raea sao "PROMPT" -o hit.wav` and `raea tokensynth "TIMBRE" --notes C3,C4 -o notes/` make sounds through the same cache a composition's render uses, and `raea sounds` searches open sample diff --git a/docs/sound.feature b/docs/sound.feature index 03366ad7..1b09281a 100644 --- a/docs/sound.feature +++ b/docs/sound.feature @@ -5,8 +5,9 @@ Feature: Sound for picture: the local models on any Mac, open libraries and pack result in drones and pads and not actually design real complex music, scoring, sound effects, diegetic and non-diegetic." - Where it stands (2026-11-16): `raea setup-audio` installs both models through tools/sao/setup.sh and - tools/tokensynth/setup.sh, and `raea sounds` reaches four libraries (sdk/raea/library/sounds/catalog.json: + Where it stands (2026-11-19): `raea setup-audio` installs both models through tools/sao/setup.py and + tools/tokensynth/setup.py, run by uv on a Mac, Windows or Linux with nothing installed first (no Homebrew, git + or FFmpeg; raea fetches uv itself when the PATH has none), and `raea sounds` reaches four libraries (sdk/raea/library/sounds/catalog.json: salamander, fluid, tonejs-instruments, vcsl). The guides teach deciding what each scene is heard as, spotting the cut and composing for it (sdk/guides/craft/motion/music.md, sound.md), and list every vetted source with its licence and how to fetch it (motion/sources.md). What those guides had to write down as a gap is below. diff --git a/io/assets/src/lib.rs b/io/assets/src/lib.rs index 26f02570..e4d8261e 100644 --- a/io/assets/src/lib.rs +++ b/io/assets/src/lib.rs @@ -6,8 +6,8 @@ //! seed, what tests use), the two local audio models, `tokensynth` (an instrument's notes, for music) and `sao` (Stable //! Audio Open on Core ML, foley). Every model runs on this computer. `lyria` (music) and `fish` (narration, through //! raea's API) are retired: a take made before plays from the index, and a new one is refused. Narration is Pocket TTS -//! (raea/speech), and Fish Audio is a `raea gen` backend like fal or Replicate (`fish:`, the user's own key). The local models install themselves on first use (tools/sao/setup.sh, -//! tools/tokensynth/setup.sh). Each made asset gets a line in `assets/made.jsonl` under its cache key, a hash of +//! (raea/speech), and Fish Audio is a `raea gen` backend like fal or Replicate (`fish:`, the user's own key). The local models install themselves on first use (tools/sao/setup.py, +//! tools/tokensynth/setup.py, run through uv on any OS: setup.rs). Each made asset gets a line in `assets/made.jsonl` under its cache key, a hash of //! (model, prompt, seed, params); `made` reads only that index, so a render or a check that resolves a make() never //! generates. @@ -24,6 +24,8 @@ pub use splat::{ensure_splat, splat_dir, splat_params}; mod local; use local::*; +mod setup; +pub use setup::{run_setup, setup_how, uv, venv_python}; #[cfg(test)] mod tests; @@ -192,11 +194,11 @@ pub fn setup_script(model: &str) -> PathBuf { dunce::canonicalize(&p).unwrap_or(p) } -/// Whether a local model can run from `dir`. SAO: the venv and either run.py with setup.sh's `.ready`, or the +/// Whether a local model can run from `dir`. SAO: the venv and either run.py with its setup's `.ready`, or the /// hand-built spike's coreml_run.py (~/.cache/raea-sao before tools/sao existed, still served as it is). TokenSynth: -/// the venv, run.py, and setup.sh's `.ready` or the fp16 runtime that every finished install has. +/// the venv, run.py, and its setup's `.ready` or the fp16 runtime that every finished install has. pub fn installed(model: &str, dir: &Path) -> bool { - let py = dir.join(".venv/bin/python").exists(); + let py = venv_python(dir).exists(); match model { "sao" => py && ((dir.join("run.py").exists() && dir.join(".ready").exists()) || dir.join("coreml_run.py").exists()), "tokensynth" => py && dir.join("run.py").exists() && (dir.join(".ready").exists() || dir.join("weights-fp16/token_synth_aug.pt").exists()), @@ -205,7 +207,7 @@ pub fn installed(model: &str, dir: &Path) -> bool { } } -/// A local model's install, set up first if it isn't there: setup.sh runs once, its progress on stderr (a render's +/// A local model's install, set up first if it isn't there: its setup runs once, its progress on stderr (a render's /// log, the CLI's terminal), unless $RAEA_AUDIO_AUTOSETUP=0, when the error names the one command that installs it. pub fn ensure_installed(model: &str) -> Result { let dir = install_dir(model); @@ -218,7 +220,7 @@ pub fn ensure_installed(model: &str) -> Result { "matchmove" => "about 250 MB", _ => "about 6 GB", }; - let how = format!("{} (installs it into {}, {size}, once)", script.display(), dir.display()); + let how = format!("{} (installs it into {}, {size}, once)", setup_how(&script), dir.display()); if std::env::var("RAEA_AUDIO_AUTOSETUP").as_deref() == Ok("0") { return Err(format!("{model} isn't installed: run {how}")); } @@ -227,14 +229,7 @@ pub fn ensure_installed(model: &str) -> Result { return Err(format!("{model} isn't installed in {}, and its setup script isn't at {}", dir.display(), script.display())); } eprintln!("raea: {model} isn't installed yet; setting it up now with {how}"); - let env = install_var(model).0; - let status = std::process::Command::new("sh") - .arg(&script) - .env(env, &dir) - // its progress on stderr: stdout is the caller's answer (a render's APPLIED line, the CLI's) - .stdout(std::io::stderr()) - .status() - .map_err(|e| format!("{model} setup: {e}"))?; + let status = run_setup(&script, install_var(model).0, &dir).map_err(|e| format!("{model} setup: {e}"))?; if !status.success() || !installed(model, &dir) { return Err(format!("{model} setup didn't finish ({status}): run {how}")); } @@ -307,7 +302,7 @@ impl Store { std::fs::create_dir_all(&self.dir).map_err(|e| format!("{}: {e}", self.dir.display()))?; let out = self.dir.join(format!(".matchmove-{key}.part")); let _ = std::fs::remove_dir_all(&out); - let mut cmd = std::process::Command::new(dir.join(".venv/bin/python")); + let mut cmd = std::process::Command::new(venv_python(&dir)); // the repo's run.py when raea's files are here (so an edit to it is used without reinstalling), else the installed copy let repo = setup_script("matchmove").with_file_name("run.py"); cmd.arg("-I").arg(if repo.exists() { repo } else { dir.join("run.py") }).arg("--video").arg(clip).arg("--out").arg(&out); diff --git a/io/assets/src/local.rs b/io/assets/src/local.rs index 5226feb3..8934d263 100644 --- a/io/assets/src/local.rs +++ b/io/assets/src/local.rs @@ -12,7 +12,7 @@ pub(crate) fn sao(prompt: &str, seed: u64, params: &J, part: &Path) -> Result Re } let dir = ensure_installed("tokensynth")?; let wav = part.with_extension("wav"); - let out = std::process::Command::new(dir.join(".venv/bin/python")) + let out = std::process::Command::new(venv_python(&dir)) .arg("-I") .arg(dir.join("run.py")) .args(["--timbre", prompt, "--seed", &seed.to_string()]) diff --git a/io/assets/src/setup.rs b/io/assets/src/setup.rs new file mode 100644 index 00000000..da892656 --- /dev/null +++ b/io/assets/src/setup.rs @@ -0,0 +1,119 @@ +//! A local model's install on any OS, with nothing from Homebrew or the system's package manager: its +//! tools//setup.py, run by uv (which makes the venv and brings its own Python), and the venv's Python run after. +//! uv is the one on the PATH, else raea's own: a pinned release fetched once into ~/.cache/raea-tools/uv and checked +//! against its SHA-256 (where tools/ensure.sh puts the same uv on a Mac, so either finds the other's). Until 0.1.8 each setup was a `sh` script needing uv already installed (`brew install uv`), so Windows +//! had no way to install any local model, and a Mac without Homebrew none either. + +use super::*; + +/// The uv raea fetches when there's none on the PATH: the release every local model's setup was measured with +const UV: &str = "0.12.24"; + +/// uv's release archive for each platform raea ships on, by target, and its SHA-256 (the release's sha256.sum) +const UV_ARCHIVES: [(&str, &str); 6] = [ + ("aarch64-apple-darwin", "0c4346de7abdb49495b393b9ec809fe387aa43e586be20fecb972216c1e71732"), + ("x86_64-apple-darwin", "4fa82e37cb94767661f532b001e470b67a186c7260e305bd84ddb78fd545c0b6"), + ("aarch64-unknown-linux-gnu", "5231be65f496304623895dacdbf1de8504fec90303684bdf05805aa34414dd21"), + ("x86_64-unknown-linux-gnu", "b4dfaef47d491a7296981f8374a4595f55dbf84e8937c8ecd2983574d8bb3da6"), + ("aarch64-pc-windows-msvc", "4b783bda5cc44bbae0651a837223873a7acee31152381aef5fc86ef6bef25997"), + ("x86_64-pc-windows-msvc", "7c38608c8a18ee137d748a1773053b07ec8f3a30fab49aebaa6f4e4efeceb019"), +]; + +fn exe(name: &str) -> String { + if cfg!(windows) { format!("{name}.exe") } else { name.to_string() } +} + +fn on_path(name: &str) -> Option { + let exe = exe(name); + std::env::split_paths(&std::env::var_os("PATH")?).map(|d| d.join(&exe)).find(|p| p.is_file()) +} + +/// The Python of the venv in an install folder: .venv/bin/python, or .venv\Scripts\python.exe on Windows +pub fn venv_python(dir: &Path) -> PathBuf { + if cfg!(windows) { dir.join(".venv").join("Scripts").join("python.exe") } else { dir.join(".venv").join("bin").join("python") } +} + +/// uv: $RAEA_UV, the one on the PATH, or raea's own, fetched the first time (about 20 MB) into $RAEA_TOOLS_CACHE (else +/// ~/.cache/raea-tools)/uv, as tools/ensure.sh does +pub fn uv() -> Result { + if let Some(p) = std::env::var_os("RAEA_UV").map(PathBuf::from).filter(|p| p.is_file()) { + return Ok(p); + } + if let Some(p) = on_path("uv") { + return Ok(p); + } + let dir = raea_io_home::env_path("RAEA_TOOLS_CACHE", ".cache/raea-tools").join("uv"); + let bin = dir.join(exe("uv")); + if bin.is_file() { + return Ok(bin); + } + let (os, arch) = (std::env::consts::OS, std::env::consts::ARCH); + let target = match os { + "macos" => format!("{arch}-apple-darwin"), + "linux" => format!("{arch}-unknown-linux-gnu"), + _ => format!("{arch}-pc-windows-msvc"), + }; + let Some(&(_, sum)) = UV_ARCHIVES.iter().find(|a| a.0 == target) else { + return Err(format!("raea has no uv for {os} {arch}: install uv (https://docs.astral.sh/uv) and run again")); + }; + let name = format!("uv-{target}.{}", if os == "windows" { "zip" } else { "tar.gz" }); + raea_io_home::offline(&format!("fetching uv {UV} for a local model's setup"))?; + eprintln!("raea: fetching uv {UV} ({name}) into {}", dir.display()); + // unpacked beside it first and moved in whole, so a stopped fetch never leaves a uv that looks installed + let part = dir.with_file_name(format!("uv.part{}", std::process::id())); + let fetched = fetch_uv(&part, &name, sum, os != "windows"); + if fetched.is_ok() && !bin.is_file() { + let _ = std::fs::rename(&part, &dir); + } + let _ = std::fs::remove_dir_all(&part); + fetched?; + if bin.is_file() { Ok(bin) } else { Err(format!("couldn't put uv in {}", dir.display())) } +} + +/// uv's release archive `name` downloaded into `part` with curl, checked against `sum` and unpacked there by tar +/// (Windows' own curl.exe and tar.exe, which reads a zip: the .tar.gz holds uv-/uv, the zip uv.exe at its top) +fn fetch_uv(part: &Path, name: &str, sum: &str, strip: bool) -> Result<(), String> { + std::fs::create_dir_all(part).map_err(|e| format!("{}: {e}", part.display()))?; + let archive = part.join(name); + let url = format!("https://github.com/astral-sh/uv/releases/download/{UV}/{name}"); + let got = std::process::Command::new(exe("curl")).args(["-fsSL", "--retry", "3", "-o"]).arg(&archive).arg(&url).status(); + if !got.is_ok_and(|s| s.success()) { + return Err(format!("couldn't download {url} with curl")); + } + let bytes = std::fs::read(&archive).map_err(|e| format!("{}: {e}", archive.display()))?; + if format!("{:x}", Sha256::digest(&bytes)) != sum { + return Err(format!("{name} doesn't match its SHA-256")); + } + let mut tar = std::process::Command::new(exe("tar")); + tar.arg("-xf").arg(&archive).arg("-C").arg(part); + if strip { + tar.arg("--strip-components=1"); + } + if !tar.status().is_ok_and(|s| s.success()) { + return Err(format!("couldn't unpack {name}")); + } + std::fs::remove_file(&archive).map_err(|e| format!("{}: {e}", archive.display())) +} + +/// How a person runs a setup themselves: setup.sh on a Mac or Linux, uv on setup.py on Windows +pub fn setup_how(script: &Path) -> String { + if cfg!(windows) { format!("uv run --no-project --python 3.11 python -I {}", script.with_file_name("setup.py").display()) } else { script.display().to_string() } +} + +/// A model's setup, its progress on stderr (stdout is the caller's answer: a render's APPLIED line, the CLI's), with +/// `var` naming the install folder `dir`: tools//setup.py by uv beside `script` (its setup.sh), else the +/// setup.sh itself by sh. +pub fn run_setup(script: &Path, var: &str, dir: &Path) -> Result { + let py = script.with_file_name("setup.py"); + let mut cmd = if py.exists() { + let uv = uv()?; + let mut c = std::process::Command::new(&uv); + c.args(["run", "-q", "--no-project", "--python", "3.11", "python", "-I"]).arg(&py).env("UV", &uv); + c + } else { + let mut c = std::process::Command::new("sh"); + c.arg(script); + c + }; + cmd.env(var, dir).stdout(std::io::stderr()).status().map_err(|e| format!("{}: {e}", py.display())) +} diff --git a/io/assets/src/splat.rs b/io/assets/src/splat.rs index 245a2491..14257910 100644 --- a/io/assets/src/splat.rs +++ b/io/assets/src/splat.rs @@ -14,8 +14,13 @@ pub fn splat_dir() -> PathBuf { raea_io_home::env_path("RAEA_SPLAT", ".cache/raea-splat") } +/// Brush's trainer in the install: brush/brush_app (brush_app.exe on Windows) +fn brush(dir: &Path) -> PathBuf { + dir.join("brush").join(if cfg!(windows) { "brush_app.exe" } else { "brush_app" }) +} + fn splat_installed(dir: &Path) -> bool { - dir.join(".venv/bin/python").exists() && dir.join("run.py").exists() && dir.join(".ready").exists() && dir.join("brush/brush_app").exists() + venv_python(dir).exists() && dir.join("run.py").exists() && dir.join(".ready").exists() && brush(dir).exists() } /// The trainer's install, set up first if it isn't there (tools/splat/setup.sh, about 360 MB, once), unless @@ -26,7 +31,7 @@ pub fn ensure_splat() -> Result { return Ok(dir); } let script = setup_script("splat"); - let how = format!("{} (installs it into {}, about 360 MB, once)", script.display(), dir.display()); + let how = format!("{} (installs it into {}, about 360 MB, once)", setup_how(&script), dir.display()); if std::env::var("RAEA_AUDIO_AUTOSETUP").as_deref() == Ok("0") { return Err(format!("the splat trainer isn't installed: run {how}")); } @@ -34,12 +39,7 @@ pub fn ensure_splat() -> Result { return Err(format!("the splat trainer isn't installed in {}, and its setup script isn't at {}", dir.display(), script.display())); } eprintln!("raea: the splat trainer isn't installed yet; setting it up now with {how}"); - let status = std::process::Command::new("sh") - .arg(&script) - .env("RAEA_SPLAT", &dir) - .stdout(std::io::stderr()) - .status() - .map_err(|e| format!("splat setup: {e}"))?; + let status = run_setup(&script, "RAEA_SPLAT", &dir).map_err(|e| format!("splat setup: {e}"))?; if !status.success() || !splat_installed(&dir) { return Err(format!("splat setup didn't finish ({status}): run {how}")); } @@ -127,8 +127,8 @@ impl Store { std::fs::create_dir_all(&self.dir).map_err(|e| format!("{}: {e}", self.dir.display()))?; let out = self.dir.join(format!(".splat-{key}.part")); let _ = std::fs::remove_dir_all(&out); - let mut cmd = std::process::Command::new(dir.join(".venv/bin/python")); - cmd.arg("-I").arg(dir.join("run.py")).arg("--input").arg(input).arg("--out").arg(&out).arg("--brush").arg(dir.join("brush/brush_app")); + let mut cmd = std::process::Command::new(venv_python(&dir)); + cmd.arg("-I").arg(dir.join("run.py")).arg("--input").arg(input).arg("--out").arg(&out).arg("--brush").arg(brush(&dir)); for (k, flag) in [("frames", "--frames"), ("steps", "--steps"), ("max_res", "--max-res"), ("max_splats", "--max-splats")] { cmd.args([flag.to_string(), (params[k].as_f64().unwrap_or(0.0) as i64).to_string()]); } diff --git a/systems/bake/Cargo.toml b/systems/bake/Cargo.toml index f03f90b4..389caea7 100644 --- a/systems/bake/Cargo.toml +++ b/systems/bake/Cargo.toml @@ -8,6 +8,8 @@ description = "Rigid bodies, glTF parts and goo baked once per request into a co [dependencies] raea-io-home = { path = "../../io/home" } +# a local tool's install on any OS (uv, the venv's Python): the goo solver's +raea-io-assets = { path = "../../io/assets" } serde_json = "1" sha2 = "0.10" image = { version = "0.25", default-features = false, features = ["png"] } diff --git a/systems/bake/src/goo.rs b/systems/bake/src/goo.rs index 91e3cbfb..aecdc0ab 100644 --- a/systems/bake/src/goo.rs +++ b/systems/bake/src/goo.rs @@ -16,7 +16,7 @@ const GOO_CODE: [&str; 2] = ["run.py", "sph.py"]; fn ensure_goo(script: &Path) -> R { let dir = goo_dir(); - let ready = |d: &Path| d.join(".venv/bin/python").exists() && GOO_CODE.iter().all(|f| d.join(f).exists()) && d.join(".ready").exists(); + let ready = |d: &Path| raea_io_assets::venv_python(d).exists() && GOO_CODE.iter().all(|f| d.join(f).exists()) && d.join(".ready").exists(); // a newer script in the tree than the install's: set up again (it copies them) let stale = GOO_CODE.iter().any(|f| { std::fs::metadata(script.with_file_name(f)).and_then(|m| m.modified()).ok() @@ -27,12 +27,12 @@ fn ensure_goo(script: &Path) -> R { return Ok(dir); } if std::env::var("RAEA_AUDIO_AUTOSETUP").as_deref() == Ok("0") && !ready(&dir) { - return Err(format!("the goo solver isn't installed: run {}", script.display())); + return Err(format!("the goo solver isn't installed: run {}", raea_io_assets::setup_how(script))); } eprintln!("raea: setting up the goo solver ({} into {})", script.display(), dir.display()); - let st = std::process::Command::new("sh").arg(script).env("RAEA_GOO", &dir).stdout(std::io::stderr()).status().map_err(|e| format!("goo setup: {e}"))?; + let st = raea_io_assets::run_setup(script, "RAEA_GOO", &dir).map_err(|e| format!("goo setup: {e}"))?; if !st.success() || !ready(&dir) { - return Err(format!("goo setup didn't finish ({st}): run {}", script.display())); + return Err(format!("goo setup didn't finish ({st}): run {}", raea_io_assets::setup_how(script))); } Ok(dir) } @@ -85,7 +85,7 @@ pub fn goo(at: &Place, v: &Value) -> R { let t0 = std::time::Instant::now(); eprintln!("raea: baking goo ({} objects, {} frames) -> {}", spec["objects"].as_array().map(|a| a.len()).unwrap_or(0), spec["frames"], out.display()); // (a bake is made once: a render's wait for the frame's commit doesn't count it, track::making) - let st = making(|| std::process::Command::new(tool.join(".venv/bin/python")) + let st = making(|| std::process::Command::new(raea_io_assets::venv_python(&tool)) .arg("-I").arg(tool.join("run.py")).arg(&sp).arg(&tmp) .stdout(std::io::stderr()) .status()) diff --git a/tools/goo/run.py b/tools/goo/run.py index d3812d0f..b2f9bbc0 100644 --- a/tools/goo/run.py +++ b/tools/goo/run.py @@ -435,7 +435,10 @@ def main(): import taichi as ti kw = {"cpu_max_num_threads": 1} if cpu else {} - ti.init(arch=ti.cpu if cpu else ti.metal, default_fp=ti.f32, offline_cache=True, log_level=ti.WARN, random_seed=seed, **kw) + # Metal on a Mac; elsewhere Taichi's best GPU (CUDA, Vulkan, OpenGL), and the CPU where there's none + ti.init(arch=ti.cpu if cpu else (ti.metal if sys.platform == "darwin" else ti.gpu), default_fp=ti.f32, offline_cache=True, + log_level=ti.WARN, random_seed=seed, **kw) + backend = "cpu" if cpu else ti.lang.impl.current_cfg().arch.name dims = np.ceil((bmax - bmin) / dx).astype(int) + 1 nx, ny, nz = (int(d) for d in dims) say(f"grid {nx}x{ny}x{nz}") @@ -809,7 +812,7 @@ def collect(): stats = {"particles": int(N), "objects": stats_obj, "frames": frames, "fps": fps, "dx": dx, "substeps": substeps_total, "simSeconds": round(sim_s, 2), "surfaceSeconds": round(surf_s, 2), "seconds": round(time.time() - T0, 2), "bytes": nbytes, "maxTriangles": int(max(tri_counts) if tri_counts else 0), - "glb": glb_written, "backend": "cpu" if cpu else "metal"} + "glb": glb_written, "backend": backend} Path(str(out) + ".json").write_text(json.dumps(stats, indent=1)) say(f"wrote {out} ({nbytes / 1e6:.1f} MB, {frames} frames, max {stats['maxTriangles']} triangles) in {stats['seconds']}s") diff --git a/tools/goo/setup.py b/tools/goo/setup.py new file mode 100644 index 00000000..bd781655 --- /dev/null +++ b/tools/goo/setup.py @@ -0,0 +1,32 @@ +"""Goo for raea's physics (, the `physics.goo` native call): objects melted into viscous goo by an MLS-MPM solver +(Taichi: Metal on a Mac, CUDA or Vulkan elsewhere, else the CPU; Apache-2.0), each frame's surface reconstructed +(marching cubes, scikit-image: BSD) into a mesh sequence (.rseq) the engine plays back by frame; thin liquids +("solver": "sph") by SPlisHSPlasH (MIT) instead. run.py and sph.py are copied in beside the venv. Installs into +$RAEA_GOO (default ~/.cache/raea-goo); raea runs this itself the first time a goo bake is asked for +(tools/local_setup.py: how, on any OS). pySPlisHSPlasH is pinned: 2.19.0 hangs after its first step on a Mac +(2026-10-20); 2.17.0 runs. Size: about 600 MB (the venv). No model weights.""" + +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +from local_setup import Setup # noqa: E402 + +with Setup("goo", "RAEA_GOO", "raea-goo") as s: + Path(".ready").unlink(missing_ok=True) + # uv's own Python, never a framework build (Homebrew's): SPlisHSPlasH finds its emitters' meshes beside the running + # executable (the venv's bin/resources), and a framework Python runs as Python.app inside its own install + framework = "import sys, sysconfig; sys.exit(1 if sysconfig.get_config_var('PYTHONFRAMEWORK') else 0)" + if s.python.exists() and subprocess.run([str(s.python), "-c", framework]).returncode != 0: + s.say("remaking the venv on uv's Python (this one is a framework build)") + shutil.rmtree(".venv") + s.venv("3.12", ["taichi", "numpy", "scipy", "trimesh", "scikit-image", "zstandard", "pillow", "pySPlisHSPlasH==2.17.0"], + imports=("taichi", "numpy", "scipy", "trimesh", "skimage", "zstandard", "PIL", "pysplishsplash"), managed=True) + s.run([s.python, "-c", "import taichi, numpy, trimesh, skimage; print('taichi', taichi.__version__, 'numpy', numpy.__version__, 'trimesh', trimesh.__version__, 'skimage', skimage.__version__)"], + stdout=sys.stderr) + for f in ("run.py", "sph.py"): + shutil.copy2(HERE / f, f) + s.ready(".venv") diff --git a/tools/goo/setup.sh b/tools/goo/setup.sh index c6dac935..cc023767 100755 --- a/tools/goo/setup.sh +++ b/tools/goo/setup.sh @@ -1,39 +1,16 @@ #!/bin/sh -# Goo for raea's physics (, the `physics.goo` native call): objects melted into viscous goo by an MLS-MPM solver -# (Taichi on Metal, Apache-2.0), each frame's surface reconstructed (marching cubes, scikit-image: BSD) into a mesh -# sequence (.rseq) the engine plays back by frame; thin liquids ("solver": "sph") by SPlisHSPlasH (MIT) instead. -# run.py and sph.py are copied in beside the venv. Installs into $RAEA_GOO -# (default ~/.cache/raea-goo), outside the repo. raea runs this itself the first time a goo bake is asked for. -# pySPlisHSPlasH is pinned: 2.19.0 hangs after its first step on this Mac (2026-10-20); 2.17.0 runs. -# Size: about 600 MB (the venv: taichi, numpy, scipy, trimesh, scikit-image, pySPlisHSPlasH). No model weights. +# goo's setup on a Mac or Linux: setup.py beside this (what it installs, where, how big), run through uv as raea runs it +# on the model's first use (raea_io_assets::run_setup). On Windows: +# uv run --no-project --python 3.11 python -I tools\goo\setup.py +# uv: $UV, the one on the PATH, raea's own (~/.cache/raea-tools/uv), else on an Apple-silicon Mac tools/ensure.sh +# fetches that one now. # tools/goo/setup.sh set -e -DIR=${RAEA_GOO:-$HOME/.cache/raea-goo} -HERE=$(cd "$(dirname "$0")" && pwd) -mkdir -p "$DIR" -DIR=$(cd "$DIR" && pwd) -if ! mkdir "$DIR/.setup-lock" 2>/dev/null; then - echo "goo setup: another setup is running in $DIR (remove $DIR/.setup-lock if it isn't)" >&2 - exit 75 -fi -trap 'rm -rf "$DIR/.setup-lock"' EXIT -trap 'exit 130' INT TERM -cd "$DIR" -say() { echo "goo setup: $*" >&2; } -rm -f .ready -# uv's own Python, not a framework build (Homebrew's): SPlisHSPlasH finds its emitters' meshes beside the running -# executable (the venv's bin/resources), and a framework Python runs as Python.app inside its own install -if [ -x .venv/bin/python ] && ! .venv/bin/python -c "import sys, sysconfig; sys.exit(1 if sysconfig.get_config_var('PYTHONFRAMEWORK') else 0)"; then - say "remaking the venv on uv's Python (this one is a framework build)" - rm -rf .venv -fi -if ! .venv/bin/python -c "import taichi, numpy, scipy, trimesh, skimage, zstandard, PIL, pysplishsplash" 2>/dev/null; then - say "making the venv" - [ -x .venv/bin/python ] || uv venv -q --python-preference only-managed --python 3.12 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python taichi numpy scipy trimesh scikit-image zstandard pillow "pySPlisHSPlasH==2.17.0" -fi -.venv/bin/python -c "import taichi, numpy, trimesh, skimage; print('taichi', taichi.__version__, 'numpy', numpy.__version__, 'trimesh', trimesh.__version__, 'skimage', skimage.__version__)" >&2 -cp "$HERE/run.py" "$HERE/sph.py" "$DIR/" -date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" -du -sh "$DIR" >&2 -say "ready in $DIR" +here=$(cd "$(dirname "$0")" && pwd) +own=${RAEA_TOOLS_CACHE:-$HOME/.cache/raea-tools}/uv/uv +UV=${UV:-$(command -v uv || true)} +[ -n "$UV" ] || [ ! -x "$own" ] || UV=$own +[ -n "$UV" ] || [ "$(uname -sm)" != "Darwin arm64" ] || UV=$("$here/../ensure.sh" uv) +[ -n "$UV" ] || { echo "goo setup: needs uv: install it (https://docs.astral.sh/uv), or let raea fetch its own on the model's first use" >&2; exit 1; } +export UV +exec "$UV" run -q --no-project --python 3.11 python -I "$here/setup.py" "$@" diff --git a/tools/local_setup.py b/tools/local_setup.py new file mode 100644 index 00000000..0342bae4 --- /dev/null +++ b/tools/local_setup.py @@ -0,0 +1,138 @@ +"""What every local model's setup.py shares (tools/sao, tokensynth, matchmove, splat, goo). + +raea runs a model's setup.py the first time the model is asked for, through uv on any OS +(`uv run --no-project --python 3.11 python -I tools//setup.py`, raea_io_assets::run_setup), with the install +folder in the model's variable ($RAEA_SAO, ...) and uv in $UV; the setup.sh beside each does the same by hand. Nothing +here needs Homebrew, a system Python or a POSIX shell: files come through curl (on a Mac, Linux, and Windows 10 and +later), checked against their SHA-256 or copied from another install; the venv is uv's, with its own Python; and that +Python is .venv/bin/python, or .venv/Scripts/python.exe on Windows. Standard library only: it runs before any venv.""" + +import hashlib +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +WINDOWS = os.name == "nt" +MAC = sys.platform == "darwin" + +# the footage tools' venv (matchmove and splat), at the versions measured (2026-10-20; PyAV 2026-11-19), one set so uv's +# cache serves both: COLMAP, OpenCV, Pillow, and PyAV, whose wheels carry FFmpeg's own libraries (no FFmpeg installed) +FOOTAGE = ["pycolmap==4.2.1", "numpy==2.4.6", "opencv-python-headless==5.0.0.93", "pillow==13.0.0", "av==18.1.0"] + + +def digest(path): + """A file's SHA-256, hex""" + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + return h.hexdigest() + + +def low(): + """Run a heavy step at a low priority where the OS has one (nice 15), so a render or a person's work goes first""" + return {} if WINDOWS else {"preexec_fn": lambda: os.nice(15)} + + +class Setup: + """An install folder, held by one setup at a time: `with Setup("sao", "RAEA_SAO", "raea-sao") as s:` works in + the folder, and a second setup started meanwhile exits 75 naming the lock.""" + + def __init__(self, name, var, folder): + self.name = name + self.dir = Path(os.environ.get(var) or Path.home() / ".cache" / folder).expanduser().resolve() + # another install to copy from instead of downloading: $_FROM, else the default folder when installing elsewhere + other = Path(os.environ.get(f"{var}_FROM") or Path.home() / ".cache" / folder).expanduser() + self.other = other.resolve() if other.is_dir() and other.resolve() != self.dir else None + self.uv = os.environ.get("UV") or shutil.which("uv") + if not self.uv: + sys.exit(f"{name} setup: needs uv: `raea setup-audio` fetches raea's own, or install it (https://docs.astral.sh/uv)") + self.lock = self.dir / ".setup-lock" + + def __enter__(self): + self.dir.mkdir(parents=True, exist_ok=True) + try: + self.lock.mkdir() + except FileExistsError: + print(f"{self.name} setup: another setup is running in {self.dir} (remove {self.lock} if it isn't)", file=sys.stderr) + sys.exit(75) + os.chdir(self.dir) + return self + + def __exit__(self, *exc): + shutil.rmtree(self.lock, ignore_errors=True) + + def say(self, *words): + print(f"{self.name} setup:", *words, file=sys.stderr, flush=True) + + @property + def python(self): + return self.dir / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python") + + def run(self, cmd, **kw): + subprocess.run([str(c) for c in cmd], check=True, **kw) + + def imports(self, *modules): + """Whether the venv's Python imports every one of `modules`""" + return self.python.exists() and subprocess.run([str(self.python), "-c", "import " + ", ".join(modules)], capture_output=True).returncode == 0 + + def venv(self, version, packages, imports=(), what="making the venv", managed=False): + """The venv at Python `version` with `packages`; one that already imports `imports` is left alone. `managed`: + on uv's own Python, never one the system or Homebrew has""" + if imports and self.imports(*imports): + return + self.say(what) + if not self.python.exists(): + self.run([self.uv, "venv", "-q", *(["--python-preference", "only-managed"] if managed else []), "--python", version, ".venv"]) + self.run([self.uv, "pip", "install", "-q", "--python", self.python, *packages], env={**os.environ, "VIRTUAL_ENV": str(self.dir / ".venv")}) + + def copy(self, src, dest): + """A file or folder copied: an APFS clone on a Mac (no extra disk until one side changes)""" + if MAC and subprocess.run(["cp", "-cR", str(src), str(dest)], capture_output=True).returncode == 0: + return + if Path(src).is_dir(): + shutil.copytree(src, dest) + else: + shutil.copy2(src, dest) + + def download(self, url, to): + """`url` into the file `to`, through curl (its progress bar on a terminal; in a log, a render's or the CLI's + piped, only errors), else Python's own HTTP""" + curl = shutil.which("curl") + if curl: + self.run([curl, "-fL", "--retry", "3", *(["--progress-bar"] if sys.stderr.isatty() else ["-sS"]), "-o", to, url]) + return + with urllib.request.urlopen(url, timeout=120) as r, open(to, "wb") as f: + shutil.copyfileobj(r, f, 1 << 20) + + def fetch(self, dest, url, sha256=None): + """A file of the install: kept when it's here, copied from the other install when that has it, else downloaded; + checked against `sha256` when one is given""" + dest = Path(dest) + if dest.is_file() and dest.stat().st_size > 0: + return + dest.parent.mkdir(parents=True, exist_ok=True) + part = dest.with_name(dest.name + ".part") + part.unlink(missing_ok=True) + theirs = self.other / dest if self.other else None + if theirs and theirs.is_file() and theirs.stat().st_size > 0: + self.copy(theirs, part) + else: + self.say(f"downloading {dest.as_posix()}") + self.download(url, part) + if sha256 and digest(part) != sha256: + part.unlink() + sys.exit(f"{self.name} setup: {dest.as_posix()} doesn't match its SHA-256") + os.replace(part, dest) + + def ready(self, *folders): + """The install marked whole (.ready, the time it finished), with the size of `folders`""" + (self.dir / ".ready").write_text(time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + "\n") + for f in folders: + n = sum(p.stat().st_size for p in Path(f).rglob("*") if p.is_file() and not p.is_symlink()) + self.say(f"{f}: {n / 1e6:.0f} MB") + self.say(f"ready in {self.dir}") diff --git a/tools/matchmove/run.py b/tools/matchmove/run.py index b170eb3f..8dc45d43 100644 --- a/tools/matchmove/run.py +++ b/tools/matchmove/run.py @@ -22,9 +22,48 @@ # points: [[x,y,z]...] (up to 3000), ground: {y: 0, normal: [0,1,0]}, cut: "cut/%04d.png" | null, notes: [...]} import argparse, glob, json, math, os, shutil, subprocess, sys, time +import av import numpy as np from PIL import Image, ImageFilter +# the colour matrices FFmpeg names for a picture's colorspace tag: BT.601 for bt470bg (5) and smpte170m (6), else BT.709 +# (an untagged phone or render clip is BT.709 limited range; FFmpeg alone would assume BT.601) +MATRIX = {5: "ITU601", 6: "ITU601"} + + +def footage(path): + """The clip's width, height, frame rate (ffprobe's r_frame_rate), duration and colour matrix, through PyAV (its + wheels carry FFmpeg's own libraries: no FFmpeg need be installed)""" + with av.open(path) as c: + v = c.streams.video[0] + rate = float(v.base_rate or v.average_rate or v.guessed_rate or 30) + dur = float(v.duration * v.time_base) if v.duration else (c.duration or 0) / av.time_base + return v.codec_context.width, v.codec_context.height, rate, dur, MATRIX.get(int(v.codec_context.colorspace or 2), "ITU709") + + +def decoded(path, start, seconds): + """Each frame from `start` for `seconds`, as `ffmpeg -ss START -i PATH -t SECONDS` decodes them""" + with av.open(path) as c: + v = c.streams.video[0] + v.thread_type = "AUTO" + t0 = v.start_time or 0 + if start > 0: + c.seek(t0 + int(start / v.time_base), stream=v, backward=True) + for f in c.decode(v): + if f.pts is None: + continue + t = float((f.pts - t0) * v.time_base) + if t < start - 1e-6: + continue + if t >= start + seconds - 1e-6: + break + yield f + + +def rgb(f, matrix, **size): + """A frame as RGB from limited-range YUV in `matrix` (FFmpeg's scale=in_color_matrix=M:in_range=tv,format=rgb24)""" + return f.reformat(format="rgb24", src_colorspace=matrix, src_color_range="MPEG", **size).to_ndarray() + ap = argparse.ArgumentParser() ap.add_argument("--video", required=True) ap.add_argument("--from", dest="start", type=float, default=0.0) @@ -49,17 +88,11 @@ def say(*m): os.makedirs(f"{work}/img") notes = [] -probe = json.loads(subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,r_frame_rate,duration,color_space", "-of", "json", a.video], capture_output=True, text=True, check=True).stdout)["streams"][0] -W, H = int(probe["width"]), int(probe["height"]) -num, den = probe["r_frame_rate"].split("/") -fps = float(num) / float(den) -dur = float(probe.get("duration") or 0) +W, H, fps, dur, matrix = footage(a.video) seconds = a.seconds if a.seconds > 0 else max(0.0, dur - a.start) -# the decoders' colour: BT.709 limited range unless the file says otherwise (ffmpeg would assume BT.601 when untagged) -matrix = {"bt470bg": "bt601", "smpte170m": "bt601"}.get(probe.get("color_space", ""), "bt709") sw, sh = W // 2, H // 2 -rng = ["-ss", f"{a.start:.6f}", "-i", a.video, "-t", f"{seconds:.6f}"] -subprocess.run(["ffmpeg", "-v", "error", "-y", *rng, "-vf", f"scale={sw}:{sh}:flags=area:in_color_matrix={matrix}:in_range=tv,format=rgb24", "-start_number", "1", f"{work}/img/%04d.png"], check=True) +for i, f in enumerate(decoded(a.video, a.start, seconds), 1): + Image.fromarray(rgb(f, matrix, width=sw, height=sh, interpolation="AREA")).save(f"{work}/img/{i:04d}.png", compress_level=1) frames = sorted(glob.glob(f"{work}/img/*.png")) N = len(frames) if N < 2: @@ -402,14 +435,12 @@ def blur(X): xn, yn = (u - W / 2) / fF, (v - H / 2) / fF d = 1 + k1 * (xn * xn + yn * yn) undistort = ((xn * d * fF + W / 2).astype(np.float32), (yn * d * fF + H / 2).astype(np.float32)) - raw = subprocess.run(["ffmpeg", "-v", "error", *rng, "-vf", f"scale=in_color_matrix={matrix}:in_range=tv,format=rgb24", "-f", "rawvideo", "-"], capture_output=True, check=True).stdout - fb = W * H * 3 - for i in range(min(len(mattes), len(raw) // fb)): - rgb = np.frombuffer(raw, np.uint8, fb, i * fb).reshape(H, W, 3) + for i, f in zip(range(len(mattes)), decoded(a.video, a.start, seconds)): + pic = rgb(f, matrix, interpolation="BICUBIC") m = Image.open(mattes[i]).convert("L").resize((W, H), Image.BILINEAR).filter(ImageFilter.GaussianBlur(1.2)) al = np.asarray(m).copy() al[al < 12] = 0 # the blur's faint spill (and EdgeTAM's low scores) isn't the subject - rgba = np.dstack([rgb * (al[..., None] > 0), al]).astype(np.uint8) + rgba = np.dstack([pic * (al[..., None] > 0), al]).astype(np.uint8) if undistort is not None: # the plate is drawn undistorted (k1): the cut-out is too, so they stay aligned rgba = cv2.remap(rgba, undistort[0], undistort[1], cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT, borderValue=0) Image.fromarray(rgba, "RGBA").save(f"{out}/cut/{i + 1:04d}.png", compress_level=4) diff --git a/tools/matchmove/setup.py b/tools/matchmove/setup.py new file mode 100644 index 00000000..1afb535f --- /dev/null +++ b/tools/matchmove/setup.py @@ -0,0 +1,20 @@ +"""Matchmove (camera solve) for raea-render's asset store: `Store::matchmove(clip, {from, seconds, subject, height})` +solves the camera of a stretch of footage (COLMAP through pycolmap: BSD; OpenCV for the rotation-only fallback: +Apache-2.0) and keeps the solve as JSON, cached by the clip's content and the params. run.py is copied in beside the +venv; it reads the footage through PyAV (BSD; its wheels carry FFmpeg's own libraries), so no FFmpeg need be installed. +Installs into $RAEA_MATCHMOVE (default ~/.cache/raea-matchmove); raea-render runs this the first time a solve is asked +for and nothing is installed (a first `raea track`, a ), on any OS (tools/local_setup.py). Size: the venv +only (pycolmap, numpy, opencv-python-headless, pillow, av: about 300 MB). No model weights.""" + +import shutil +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +from local_setup import FOOTAGE, Setup # noqa: E402 + +with Setup("matchmove", "RAEA_MATCHMOVE", "raea-matchmove") as s: + s.venv("3.11", FOOTAGE, imports=("pycolmap", "numpy", "cv2", "PIL", "av")) + shutil.copy2(HERE / "run.py", "run.py") + s.ready(".venv") diff --git a/tools/matchmove/setup.sh b/tools/matchmove/setup.sh index 9f9ccdd3..d0cc823e 100644 --- a/tools/matchmove/setup.sh +++ b/tools/matchmove/setup.sh @@ -1,33 +1,16 @@ #!/bin/sh -# Matchmove (camera solve) for raea-render's asset store: `Store::matchmove(clip, {from, seconds, subject, height})` solves the -# camera of a stretch of footage (COLMAP through pycolmap: BSD; OpenCV for the rotation-only fallback: Apache-2.0) and -# keeps the solve as JSON, cached by the clip's content and the params. run.py is copied in beside the venv. Installs -# into $RAEA_MATCHMOVE (default ~/.cache/raea-matchmove), outside the repo, as SAO's lives in ~/.cache/raea-sao. raea-render runs -# this itself the first time a solve is asked for and nothing is installed (a first `raea track`, a ); run it -# by hand to install ahead of that. Size: the venv only (pycolmap, numpy, opencv-python-headless, pillow: about 250 MB). No model weights. +# matchmove's setup on a Mac or Linux: setup.py beside this (what it installs, where, how big), run through uv as raea runs it +# on the model's first use (raea_io_assets::run_setup). On Windows: +# uv run --no-project --python 3.11 python -I tools\matchmove\setup.py +# uv: $UV, the one on the PATH, raea's own (~/.cache/raea-tools/uv), else on an Apple-silicon Mac tools/ensure.sh +# fetches that one now. # tools/matchmove/setup.sh set -e -DIR=${RAEA_MATCHMOVE:-$HOME/.cache/raea-matchmove} -HERE=$(cd "$(dirname "$0")" && pwd) -mkdir -p "$DIR" -DIR=$(cd "$DIR" && pwd) -# one setup at a time: a render and a CLI asking for the first solve together would both start one -if ! mkdir "$DIR/.setup-lock" 2>/dev/null; then - echo "matchmove setup: another setup is running in $DIR (remove $DIR/.setup-lock if it isn't)" >&2 - exit 75 -fi -trap 'rm -rf "$DIR/.setup-lock"' EXIT -trap 'exit 130' INT TERM -cd "$DIR" -say() { echo "matchmove setup: $*" >&2; } - -# the venv: only what run.py imports, at the versions measured (2026-10-20). An existing one that has them is left alone -if ! .venv/bin/python -c "import pycolmap, numpy, cv2, PIL" 2>/dev/null; then - say "making the venv" - [ -x .venv/bin/python ] || uv venv -q --python 3.11 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python "pycolmap==4.2.1" "numpy==2.4.6" "opencv-python-headless==5.0.0.93" "pillow==13.0.0" -fi -cp "$HERE/run.py" "$DIR/run.py" -date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" -du -sh "$DIR/.venv" >&2 -say "ready in $DIR" +here=$(cd "$(dirname "$0")" && pwd) +own=${RAEA_TOOLS_CACHE:-$HOME/.cache/raea-tools}/uv/uv +UV=${UV:-$(command -v uv || true)} +[ -n "$UV" ] || [ ! -x "$own" ] || UV=$own +[ -n "$UV" ] || [ "$(uname -sm)" != "Darwin arm64" ] || UV=$("$here/../ensure.sh" uv) +[ -n "$UV" ] || { echo "matchmove setup: needs uv: install it (https://docs.astral.sh/uv), or let raea fetch its own on the model's first use" >&2; exit 1; } +export UV +exec "$UV" run -q --no-project --python 3.11 python -I "$here/setup.py" "$@" diff --git a/tools/release.sh b/tools/release.sh index 6546e75c..498eff60 100755 --- a/tools/release.sh +++ b/tools/release.sh @@ -160,7 +160,7 @@ done # (@raea/ui, the page's DOM every render goes through, isn't an @raea/ path, so it's named here) git archive "$ref" sdk/tsconfig.base.json sdk/ui/src sdk/ui/package.json sdk/ui/bevy sdk/raea/src sdk/raea/library sdk/raea/package.json sdk/react/js/src sdk/react/js/lite sdk/react/js/shims sdk/react/js/package.json sdk/react/js/tsconfig.json sdk/shell/src \ "${pkgs[@]}" \ - tools/tokensynth tools/sao tools/goo tools/splat tools/matchmove tools/clothes tools/garments tools/ensure.sh sdk/guides/craft docs/sdk-coverage.md engine/cli/template/theme.css ':(glob)engine/cli/src/*.ts' \ + tools/tokensynth tools/sao tools/goo tools/splat tools/matchmove tools/local_setup.py tools/clothes tools/garments tools/ensure.sh sdk/guides/craft docs/sdk-coverage.md engine/cli/template/theme.css ':(glob)engine/cli/src/*.ts' \ | tar -x -C "$stage/share/raea" # a folder copied as it is here (links kept, as rsync -a; tar, since Git's bash on Windows has no rsync) copy() { mkdir -p "$2" && tar -C "$(dirname "$1")" --exclude .cache -cf - "$(basename "$1")" | tar -C "$2" -xf -; } @@ -184,7 +184,7 @@ for s in "$skills"/skills/*/SKILL.md; do done # the runtime's packages, as installed here copy sdk/react/js/node_modules "$stage/share/raea/sdk/react/js" -rm -rf "$stage/share/raea/tools/tokensynth/__pycache__" "$skills" +rm -rf "$stage/share/raea/tools"/*/__pycache__ "$skills" printf '%s\n' "$version" > "$stage/share/raea/VERSION" # each skill's front matter names the raea it shipped with, so an agent holding an older or newer copy than its raea # can tell, and reads `raea skill` instead (docs/skills.feature SK6; `raea skills install` stamps the same way) diff --git a/tools/sao/run.py b/tools/sao/run.py index 671627df..175f4f28 100644 --- a/tools/sao/run.py +++ b/tools/sao/run.py @@ -1,7 +1,7 @@ """Stable Audio Open Small on Core ML (a Mac) or ONNX Runtime (Linux, Windows, raea's cloud GPUs): one sound for a prompt and a seed, as a 44.1 kHz stereo WAV peak-normalised to full scale. rdb's asset store runs it for make("sao", prompt, seed, {seconds}) and keeps the bytes under their hash (sdk/react/docs/audio-stack.md, "Generated -sources"); tools/sao/setup.sh installs it beside the models. +sources"); tools/sao/setup.py installs it beside the models. run.py --prompt TEXT --wav OUT.wav [--seed 1] [--seconds 4] [--steps 8] [--sampler pingpong|euler] [--backend coreml|onnx] [--dit-units cpu|gpu|all] [--device auto|cpu|cuda] [--times] [--dir DIR] @@ -21,7 +21,7 @@ This is ~/.cache/raea-sao/coreml_run.py (the 2026-10-15 spike) cut to what inference needs: numpy, coremltools and Hugging Face's `tokenizers` reading t5-base's tokenizer.json from the install, so a run needs neither torch nor stable-audio-tools nor the network. Same prompt and seed, same samples as the spike's runner (checked 2026-10-19).""" -import argparse, json, os, shutil, subprocess, sys, time +import argparse, json, os, shutil, sys, time, wave SR, LAT, AUDIO = 44100, 256, 524288 TOKENS = 64 @@ -90,7 +90,7 @@ def onnx(a, np): device = a.device if a.device != "auto" else ("cuda" if "CUDAExecutionProvider" in have else "cpu") if device == "cuda": if "CUDAExecutionProvider" not in have: - raise SystemExit(f"sao: this onnxruntime has no CUDA ({', '.join(have)}): run tools/sao/setup.sh on a machine with an NVIDIA GPU") + raise SystemExit(f"sao: this onnxruntime has no CUDA ({', '.join(have)}): install SAO again with RAEA_SAO_CUDA=1 (raea setup-audio sao) on a machine with an NVIDIA GPU") # the CUDA and cuDNN libraries from their wheels (onnxruntime-gpu[cuda,cudnn]), not the system's if hasattr(ort, "preload_dlls"): ort.preload_dlls() @@ -164,8 +164,13 @@ def main(): audio = np.nan_to_num(audio) audio = audio / max(np.abs(audio).max(), 1e-6) pcm = (audio.T * 32767).astype("/dev/null && pwd)" = "$DIR" ] && FROM= -# one setup at a time: a render and a CLI asking for the first sound together would both start one -if ! mkdir "$DIR/.setup-lock" 2>/dev/null; then - echo "sao setup: another setup is running in $DIR (remove $DIR/.setup-lock if it isn't)" >&2 - exit 75 -fi -trap 'rm -rf "$DIR/.setup-lock"' EXIT -trap 'exit 130' INT TERM -cd "$DIR" -say() { echo "sao setup: $*" >&2; } -# a file's SHA-256 (sha256sum on Linux, shasum on a Mac) -sha256() { - if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | cut -d' ' -f1; else shasum -a 256 "$1" | cut -d' ' -f1; fi -} - -# a file of the installation: kept if it's here, cloned from $FROM if that has it, else downloaded; checked against -# its SHA-256 when one is given. fetch DEST URL [SHA256] -fetch() { - dest=$1 url=$2 sum=$3 - [ -s "$dest" ] && return 0 - mkdir -p "$(dirname "$dest")" - if [ -n "$FROM" ] && [ -s "$FROM/$dest" ]; then - cp -c "$FROM/$dest" "$dest.part" 2>/dev/null || cp "$FROM/$dest" "$dest.part" - else - say "downloading $dest" - curl -fL --retry 3 --progress-bar -o "$dest.part" "$url" - fi - if [ -n "$sum" ] && [ "$(sha256 "$dest.part")" != "$sum" ]; then - rm -f "$dest.part" - say "$dest doesn't match its SHA-256" - exit 1 - fi - mv -f "$dest.part" "$dest" -} - -# the venv: only what run.py imports. An existing one that already has them is left alone -if [ "$BACKEND" = onnx ]; then - runtime="onnxruntime==1.23.2" - # an NVIDIA GPU: the CUDA build, with CUDA's and cuDNN's libraries as wheels (run.py preloads them) - command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1 && runtime="onnxruntime-gpu[cuda,cudnn]==1.23.2" - [ -n "${RAEA_SAO_CUDA:-}" ] && runtime="onnxruntime-gpu[cuda,cudnn]==1.23.2" - if ! .venv/bin/python -c "import numpy, onnxruntime, tokenizers" 2>/dev/null; then - say "making the venv ($runtime)" - [ -x .venv/bin/python ] || uv venv -q --python 3.11 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python "numpy==2.4.6" "tokenizers==0.23.3" "$runtime" - fi -elif ! .venv/bin/python -c "import numpy, coremltools, tokenizers" 2>/dev/null; then - say "making the venv" - [ -x .venv/bin/python ] || uv venv -q --python 3.11 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python "numpy==2.4.6" "coremltools==9.0" "tokenizers==0.23.3" -fi - -# the tokenizer from the install, never the Hugging Face cache (the spike's runner read ~/.cache/huggingface) -fetch t5-tokenizer/tokenizer.json "https://huggingface.co/google-t5/t5-base/resolve/$T5REV/tokenizer.json" \ - d2acde0d8d71dd30a711834b07781b9c89feaac33fd332f60507699282740066 - -if [ "$BACKEND" = onnx ]; then - # the graphs at a pinned revision, each against the SHA-256 its model card publishes; its licence and NOTICE (what - # the conversion changed) beside them - O=https://huggingface.co/$ONNXREPO/resolve/$ONNXREV - fetch onnx/cond.onnx "$O/cond.onnx" 847822809975d0c36a665594fad0312c4c225147910657820e9936b09014629c - fetch onnx/dit.onnx "$O/dit.onnx" 8e9c8b14baac4a26159d5ecf3d365ef5a097f321541f9548d62d136354a23d8b - fetch onnx/dec.onnx "$O/dec.onnx" 96029859b4d563f3aff266e97e715190e8d53278c6e27ab94095535a0da4c911 - fetch onnx/LICENSE "$O/LICENSE" - fetch onnx/NOTICE "$O/NOTICE" - cp "$HERE/run.py" "$DIR/run.py" - date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" - du -sh "$DIR/onnx" "$DIR/.venv" >&2 - say "ready in $DIR (ONNX)" - exit 0 -fi - -HF=https://huggingface.co/$REPO/resolve/$REV -P=Data/com.apple.CoreML -while read -r name mlmodel weights; do - fetch "coreml-ref/StableAudio$name.mlpackage/Manifest.json" "$HF/StableAudio$name.mlpackage/Manifest.json" - fetch "coreml-ref/StableAudio$name.mlpackage/$P/model.mlmodel" "$HF/StableAudio$name.mlpackage/$P/model.mlmodel" "$mlmodel" - fetch "coreml-ref/StableAudio$name.mlpackage/$P/weights/weight.bin" "$HF/StableAudio$name.mlpackage/$P/weights/weight.bin" "$weights" -done </dev/null || cp -R "$FROM/$T5" "$PART" - else - # torch, transformers and t5-base's weights only for this one conversion, in uv's throwaway environment and a - # Hugging Face cache that's deleted after (on failure too, by the trap): none of it stays in the install - say "converting t5-base's encoder to Core ML at fp32 (a one-time ~1 GB download, deleted after)" - trap 'rm -rf "$DIR/.setup-lock" "$DIR/.t5-hf"' EXIT - HF_HOME="$DIR/.t5-hf" nice -n 15 uv run -q --no-project --python 3.11 --with "torch==2.7.1" --with "transformers==5.19.0" \ - --with "coremltools==9.0" --with "numpy==2.4.6" python -I "$HERE/t5_to_coreml.py" --revision "$T5REV" \ - --tokenizer t5-tokenizer/tokenizer.json --out "$PART" - rm -rf .t5-hf - fi - mv "$PART" "$T5" -fi - -cp "$HERE/run.py" "$DIR/run.py" -date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" -du -sh "$DIR/coreml-ref" "$DIR/coreml" "$DIR/.venv" >&2 -say "ready in $DIR" +here=$(cd "$(dirname "$0")" && pwd) +own=${RAEA_TOOLS_CACHE:-$HOME/.cache/raea-tools}/uv/uv +UV=${UV:-$(command -v uv || true)} +[ -n "$UV" ] || [ ! -x "$own" ] || UV=$own +[ -n "$UV" ] || [ "$(uname -sm)" != "Darwin arm64" ] || UV=$("$here/../ensure.sh" uv) +[ -n "$UV" ] || { echo "sao setup: needs uv: install it (https://docs.astral.sh/uv), or let raea fetch its own on the model's first use" >&2; exit 1; } +export UV +exec "$UV" run -q --no-project --python 3.11 python -I "$here/setup.py" "$@" diff --git a/tools/splat/run.py b/tools/splat/run.py index c26e8be2..efca6e9d 100644 --- a/tools/splat/run.py +++ b/tools/splat/run.py @@ -24,9 +24,43 @@ # average. import argparse, glob, json, math, os, re, shutil, subprocess, sys, time +import av import numpy as np from PIL import Image +# the colour matrices FFmpeg names for a picture's colorspace tag: BT.601 for bt470bg (5) and smpte170m (6), else BT.709 +# (an untagged phone or render clip is BT.709 limited range; FFmpeg alone would assume BT.601). As matchmove's run.py. +MATRIX = {5: "ITU601", 6: "ITU601"} + + +def footage(path): + """The clip's width, height, frame rate (ffprobe's r_frame_rate), duration and colour matrix, through PyAV (its + wheels carry FFmpeg's own libraries: no FFmpeg need be installed)""" + with av.open(path) as c: + v = c.streams.video[0] + rate = float(v.base_rate or v.average_rate or v.guessed_rate or 30) + dur = float(v.duration * v.time_base) if v.duration else (c.duration or 0) / av.time_base + return v.codec_context.width, v.codec_context.height, rate, dur, MATRIX.get(int(v.codec_context.colorspace or 2), "ITU709") + + +def decoded(path, start, seconds): + """Each frame from `start` for `seconds`, as `ffmpeg -ss START -i PATH -t SECONDS` decodes them""" + with av.open(path) as c: + v = c.streams.video[0] + v.thread_type = "AUTO" + t0 = v.start_time or 0 + if start > 0: + c.seek(t0 + int(start / v.time_base), stream=v, backward=True) + for f in c.decode(v): + if f.pts is None: + continue + t = float((f.pts - t0) * v.time_base) + if t < start - 1e-6: + continue + if t >= start + seconds - 1e-6: + break + yield f + ap = argparse.ArgumentParser() ap.add_argument("--input", required=True) ap.add_argument("--out", required=True) @@ -80,21 +114,18 @@ def say(*m): say(f"{len(meta)} pictures from {src}") else: video = True - probe = json.loads(subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height,r_frame_rate,duration,color_space", "-of", "json", src], capture_output=True, text=True, check=True).stdout)["streams"][0] - W, H = int(probe["width"]), int(probe["height"]) - num, den = probe["r_frame_rate"].split("/") - fps = float(num) / float(den) - dur = float(probe.get("duration") or 0) + W, H, fps, dur, matrix = footage(src) seconds = a.seconds if a.seconds > 0 else max(0.0, dur - a.start) - matrix = {"bt470bg": "bt601", "smpte170m": "bt601"}.get(probe.get("color_space", ""), "bt709") s = min(1.0, a.max_res / max(W, H)) sw, sh = round(W * s / 2) * 2, round(H * s / 2) * 2 # three candidates per picture kept (all the source's frames when it has fewer), the sharpest of each window wins total = max(1, int(seconds * fps)) step = max(1, total // (N * 3)) - rng = ["-ss", f"{a.start:.6f}", "-i", src, "-t", f"{seconds:.6f}"] - vf = f"select='not(mod(n\\,{step}))',scale={sw}:{sh}:flags=lanczos:in_color_matrix={matrix}:in_range=tv,format=rgb24" - subprocess.run(["ffmpeg", "-v", "error", "-y", *rng, "-vf", vf, "-fps_mode", "passthrough", "-start_number", "0", "-q:v", "2", f"{work}/cand/%05d.jpg"], check=True) + # every step-th frame, scaled with Lanczos from limited-range YUV (FFmpeg's select + scale, through PyAV) + for n, f in enumerate(decoded(src, a.start, seconds)): + if n % step == 0: + pic = f.reformat(width=sw, height=sh, format="rgb24", src_colorspace=matrix, src_color_range="MPEG", interpolation="LANCZOS").to_ndarray() + Image.fromarray(pic).save(f"{work}/cand/{n // step:05d}.jpg", quality=95, subsampling=0) # candidates as JPEG (PNG cost ~1.7 GB for 150 frames' worth); the winners are written back out as PNG cands = sorted(glob.glob(f"{work}/cand/*.jpg")) if len(cands) < 3: diff --git a/tools/splat/setup.py b/tools/splat/setup.py new file mode 100644 index 00000000..1ba59eee --- /dev/null +++ b/tools/splat/setup.py @@ -0,0 +1,61 @@ +"""Splat training for raea-render's asset store: `Store::splat(input, {frames, steps, ...})` makes a 3D Gaussian splat +from footage (frames read through PyAV, then COLMAP through pycolmap: BSD, then Brush: Apache-2.0, Rust + wgpu: Metal on +a Mac, Vulkan or DX12 elsewhere) and keeps the .ply and its sidecar, cached by the input's content and the params. +run.py is copied in beside the venv; Brush is its v0.3.0 release build for this OS, checked against its SHA-256. +Installs into $RAEA_SPLAT (default ~/.cache/raea-splat); raea-render runs this the first time a splat is asked for, and +`raea splat setup` runs it on purpose, on any OS (tools/local_setup.py). Size: about 360 MB (the venv, Brush's binary). +No model weights.""" + +import platform +import shutil +import sys +import tarfile +import zipfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +from local_setup import FOOTAGE, WINDOWS, Setup # noqa: E402 + +BRUSH = "v0.3.0" +# Brush's release builds raea runs, by (OS, machine), and their SHA-256 (the release's sha256.sum) +BUILDS = { + ("Darwin", "arm64"): ("brush-app-aarch64-apple-darwin.tar.xz", "65b2631398c839be3c1d4d7160fe2326389dec87830aac0710985e6690a1048c"), + ("Linux", "x86_64"): ("brush-app-x86_64-unknown-linux-gnu.tar.xz", "4f0f9a8785d1951c62df26aae247c02c5bba32b00f40b06df4e1c9b867399e20"), + ("Windows", "AMD64"): ("brush-app-x86_64-pc-windows-msvc.zip", "b68e3e9cf052d51bf3ee30776fa5a364de7f2ba13b58443128ff797bb7bcfcd6"), +} + +with Setup("splat", "RAEA_SPLAT", "raea-splat") as s: + s.venv("3.11", FOOTAGE, imports=("pycolmap", "numpy", "cv2", "PIL", "av")) + exe = Path("brush") / ("brush_app.exe" if WINDOWS else "brush_app") + version = Path("brush/.version") + if not exe.is_file() or not version.is_file() or version.read_text().strip() != BRUSH: + build = BUILDS.get((platform.system(), platform.machine())) + if not build: + sys.exit(f"splat setup: Brush {BRUSH} has no release build for {platform.system()} {platform.machine()}: build it " + f"(cargo install --git https://github.com/ArthurBrussee/brush --tag {BRUSH} brush-app) and put brush_app in {s.dir / 'brush'}") + name, sha = build + s.say(f"fetching Brush {BRUSH}") + shutil.rmtree("brush.part", ignore_errors=True) + s.fetch(f"brush.part/{name}", f"https://github.com/ArthurBrussee/brush/releases/download/{BRUSH}/{name}", sha) + if name.endswith(".zip"): + with zipfile.ZipFile(f"brush.part/{name}") as z: + z.extractall("brush.part") + else: + with tarfile.open(f"brush.part/{name}") as t: + t.extractall("brush.part", **({"filter": "data"} if hasattr(tarfile, "data_filter") else {})) + shutil.rmtree("brush", ignore_errors=True) + Path("brush").mkdir() + found = [p for p in Path("brush.part").rglob(exe.name) if p.is_file()] + if not found: + sys.exit(f"splat setup: {name} has no {exe.name}") + shutil.move(found[0], exe) + for lic in Path("brush.part").rglob("LICENSE*"): + shutil.move(lic, Path("brush") / lic.name) + break + exe.chmod(0o755) + version.write_text(BRUSH + "\n") + shutil.rmtree("brush.part", ignore_errors=True) + s.run([exe.resolve(), "--version"], stdout=sys.stderr) + shutil.copy2(HERE / "run.py", "run.py") + s.ready(".venv", "brush") diff --git a/tools/splat/setup.sh b/tools/splat/setup.sh index 5c4b0084..8dbd352d 100644 --- a/tools/splat/setup.sh +++ b/tools/splat/setup.sh @@ -1,51 +1,16 @@ #!/bin/sh -# Splat training for raea-render's asset store: `Store::splat(input, {frames, steps, ...})` makes a 3D Gaussian splat from -# footage (frames, then COLMAP through pycolmap: BSD, then Brush: Apache-2.0, Rust + wgpu, trains on Metal) and keeps -# the .ply and its sidecar, cached by the input's content and the params. run.py is copied in beside the venv; Brush is -# its release binary (v0.3.0, aarch64-apple-darwin, checked against its SHA-256). Installs into $RAEA_SPLAT (default -# ~/.cache/raea-splat), outside the repo. raea-render runs this itself the first time a splat is asked for; `raea splat -# setup` runs it on purpose. Size: about 300 MB (the venv, Brush's 60 MB binary). No model weights. +# splat's setup on a Mac or Linux: setup.py beside this (what it installs, where, how big), run through uv as raea runs it +# on the model's first use (raea_io_assets::run_setup). On Windows: +# uv run --no-project --python 3.11 python -I tools\splat\setup.py +# uv: $UV, the one on the PATH, raea's own (~/.cache/raea-tools/uv), else on an Apple-silicon Mac tools/ensure.sh +# fetches that one now. # tools/splat/setup.sh set -e -DIR=${RAEA_SPLAT:-$HOME/.cache/raea-splat} -HERE=$(cd "$(dirname "$0")" && pwd) -mkdir -p "$DIR" -DIR=$(cd "$DIR" && pwd) -if ! mkdir "$DIR/.setup-lock" 2>/dev/null; then - echo "splat setup: another setup is running in $DIR (remove $DIR/.setup-lock if it isn't)" >&2 - exit 75 -fi -trap 'rm -rf "$DIR/.setup-lock"' EXIT -trap 'exit 130' INT TERM -cd "$DIR" -say() { echo "splat setup: $*" >&2; } - -# the venv: only what run.py imports, at matchmove's measured versions (2026-10-20), so uv's cache serves both -if ! .venv/bin/python -c "import pycolmap, numpy, cv2, PIL" 2>/dev/null; then - say "making the venv" - [ -x .venv/bin/python ] || uv venv -q --python 3.11 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python "pycolmap==4.2.1" "numpy==2.4.6" "opencv-python-headless==5.0.0.93" "pillow==13.0.0" -fi - -# Brush: the release binary for Apple silicon -BRUSH_VER=v0.3.0 -BRUSH_SHA=65b2631398c839be3c1d4d7160fe2326389dec87830aac0710985e6690a1048c -if [ "$(uname -s)-$(uname -m)" != "Darwin-arm64" ]; then - say "Brush's release binary here is for Apple silicon; elsewhere build it (cargo install --git https://github.com/ArthurBrussee/brush --tag $BRUSH_VER brush-app) and put brush_app in $DIR/brush/" -fi -if [ ! -x brush/brush_app ] || [ "$(cat brush/.version 2>/dev/null)" != "$BRUSH_VER" ]; then - say "fetching Brush $BRUSH_VER" - rm -rf brush.part && mkdir brush.part - curl -fsSL "https://github.com/ArthurBrussee/brush/releases/download/$BRUSH_VER/brush-app-aarch64-apple-darwin.tar.xz" -o brush.part/brush.tar.xz - echo "$BRUSH_SHA brush.part/brush.tar.xz" | shasum -a 256 -c - >&2 - tar xJf brush.part/brush.tar.xz -C brush.part - rm -rf brush && mkdir brush - mv brush.part/brush-app-aarch64-apple-darwin/brush_app brush.part/brush-app-aarch64-apple-darwin/LICENSE brush/ - echo "$BRUSH_VER" > brush/.version - rm -rf brush.part -fi -brush/brush_app --version >&2 -cp "$HERE/run.py" "$DIR/run.py" -date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" -du -sh "$DIR" >&2 -say "ready in $DIR" +here=$(cd "$(dirname "$0")" && pwd) +own=${RAEA_TOOLS_CACHE:-$HOME/.cache/raea-tools}/uv/uv +UV=${UV:-$(command -v uv || true)} +[ -n "$UV" ] || [ ! -x "$own" ] || UV=$own +[ -n "$UV" ] || [ "$(uname -sm)" != "Darwin arm64" ] || UV=$("$here/../ensure.sh" uv) +[ -n "$UV" ] || { echo "splat setup: needs uv: install it (https://docs.astral.sh/uv), or let raea fetch its own on the model's first use" >&2; exit 1; } +export UV +exec "$UV" run -q --no-project --python 3.11 python -I "$here/setup.py" "$@" diff --git a/tools/tokensynth/convert.py b/tools/tokensynth/convert.py index 6cea4c3f..6d0a88a9 100644 --- a/tools/tokensynth/convert.py +++ b/tools/tokensynth/convert.py @@ -1,4 +1,4 @@ -"""TokenSynth's smaller runtime, made from the fp32 weights setup.sh fetched (run at the end of setup.sh; idempotent: a +"""TokenSynth's smaller runtime, made from the fp32 weights setup.py fetched (run at the end of setup.py; idempotent: a file already made is kept, so a second run does nothing). Everything goes beside weights/, outside the repo: weights-fp16/ the transformer, its unconditional twin, DAC's quantizer + decoder and CLAP's text tower (RoBERTa and its projection, without the audio tower or the optimizer state that make up most of the 2.35 GB CLAP diff --git a/tools/tokensynth/run.py b/tools/tokensynth/run.py index 70357e8a..0ec6ef44 100644 --- a/tools/tokensynth/run.py +++ b/tools/tokensynth/run.py @@ -243,7 +243,7 @@ def __init__(self, precision, weights, device="cpu"): if precision == "coreml": missing += [f"../coreml/{p}" for p in ("dac_decoder.mlmodelc", "token_synth_step.mlmodelc") if not (self.C / p).exists()] if missing: - raise SystemExit(f"tokensynth: {precision} needs {', '.join(missing)} in {self.F} (tools/tokensynth/setup.sh runs convert.py)") + raise SystemExit(f"tokensynth: {precision} needs {', '.join(missing)} in {self.F} (tools/tokensynth/setup.py runs convert.py)") dtype = {"fp16": torch.float16, "bf16": torch.bfloat16, "coreml": torch.float16}.get(precision, torch.float32) synth = load_transformer(TokenSynth, self.F / "token_synth_aug.pt", dtype) uncond = load_transformer(TokenSynthUnconditional, self.F / "token_synth_unconditional.pt", dtype) diff --git a/tools/tokensynth/setup.py b/tools/tokensynth/setup.py new file mode 100644 index 00000000..9846413c --- /dev/null +++ b/tools/tokensynth/setup.py @@ -0,0 +1,50 @@ +"""TokenSynth (MIT code and weights: github.com/KyungsuKim42/tokensynth, huggingface.co/KyungsuKim/TokenSynth) for +raea-render's asset store: `make("tokensynth", timbre, seed, {midi, velocity, seconds, precision})` renders one note +(torch on the CPU, or CUDA where asked). Installs the code at a pinned commit, the four fp32 weights (about 4 GB) and a +venv into $RAEA_TOKENSYNTH (default ~/.cache/raea-tokensynth); convert.py then makes the smaller runtime run.py defaults +to (weights-fp16/, 1.0 GB) and, on a Mac, the Core ML models (coreml/). raea-render runs this the first time a note is +asked for, and `raea setup-audio` runs it on purpose, on any OS (tools/local_setup.py). Another install's files are +copied rather than downloaded ($RAEA_TOKENSYNTH_FROM, else ~/.cache/raea-tokensynth when installing elsewhere).""" + +import shutil +import sys +import zipfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +from local_setup import MAC, Setup, low # noqa: E402 + +# the code at the commit the spike ran (2026-10-18; the repo's last commit, 2026-06-24), as GitHub's archive: no git needed +COMMIT = "be7f5fbeef93a8a59b203945ccbabb7c9aa4f57e" +WEIGHTS = ["token_synth_aug.pt", "token_synth_unconditional.pt", "dac_weights_44khz_8kbps_0.0.1.pt", "clap_music_audioset_epoch_15_esc_90.14.pt"] + +with Setup("tokensynth", "RAEA_TOKENSYNTH", "raea-tokensynth") as s: + for d in ("weights", "hf"): + Path(d).mkdir(exist_ok=True) + if not Path("repo").is_dir(): + if s.other and (s.other / "repo").is_dir(): + s.copy(s.other / "repo", "repo") + else: + s.fetch("repo.zip", f"https://github.com/KyungsuKim42/tokensynth/archive/{COMMIT}.zip") + with zipfile.ZipFile("repo.zip") as z: + z.extractall("repo.part") + shutil.move(next(Path("repo.part").iterdir()), "repo") + shutil.rmtree("repo.part") + Path("repo.zip").unlink() + Path("repo.commit").write_text(COMMIT + "\n") + # the versions the spike ran (2026-10-18): laion-clap 1.1.6 needs transformers 4.30; scipy 1.15's wheel wouldn't + # load on a Mac. coremltools only on a Mac, for convert.py's Core ML models (run.py --precision coreml) + packages = ["torch==2.5.1", "torchaudio==2.5.1", "transformers==4.30.2", "scipy==1.14.1", *(["coremltools==8.3.0"] if MAC else []), + "laion-clap", "descript-audio-codec", "pretty_midi", "soundfile", "numpy", "-e", "repo"] + s.venv("3.11", packages, imports=("torch", "laion_clap", "dac", "tokensynth", *(["coremltools"] if MAC else [])), what="making the venv (about 1.1 GB)") + for f in WEIGHTS: + s.fetch(f"weights/{f}", f"https://huggingface.co/KyungsuKim/TokenSynth/resolve/main/{f}") + # the converted runtime from the other install, when it has it whole: convert.py then finds everything made + for d in ("weights-fp16", "coreml"): + if not Path(d).exists() and s.other and (s.other / d).is_dir(): + s.copy(s.other / d, d) + shutil.copy2(HERE / "run.py", "run.py") + # the smaller runtime run.py's other precisions read (weights-fp16/, coreml/); keeps what is already made + s.run([s.python, "-I", HERE / "convert.py", "--dir", s.dir], **low()) + s.ready("weights", "weights-fp16", ".venv") diff --git a/tools/tokensynth/setup.sh b/tools/tokensynth/setup.sh index 3013b985..3a17e55f 100755 --- a/tools/tokensynth/setup.sh +++ b/tools/tokensynth/setup.sh @@ -1,65 +1,16 @@ #!/bin/sh -# TokenSynth (MIT code and weights: github.com/KyungsuKim42/tokensynth, huggingface.co/KyungsuKim/TokenSynth) for raea-render's -# asset store: `make("tokensynth", timbre, seed, {midi, velocity, seconds, precision})` renders one note (torch on the CPU: -# MPS is 6x slower and can't run DAC's decoder on long clips), cached as PCM by everything that decides the bytes. -# convert.py then makes the smaller runtime run.py defaults to (weights-fp16/, 1.0 GB) and the Core ML models (coreml/). -# Installs into $RAEA_TOKENSYNTH (default ~/.cache/raea-tokensynth), outside the repo, as SAO's lives in ~/.cache/raea-sao. -# raea-render runs this itself the first time a note is asked for and nothing is installed; `raea setup-audio` runs it on -# purpose. -# Size: the model 713 MB, its unconditional twin 713 MB (text timbres need it), DAC 307 MB, CLAP 2.35 GB, three text -# encoders laion-clap fetches (479 MB), and the venv (1.1 GB). Local only: no API model is called. -# -# $RAEA_TOKENSYNTH_FROM names an existing install to copy the weights and the converted runtime from instead of -# downloading and converting them (default ~/.cache/raea-tokensynth when installing somewhere else). Copies are APFS -# clones (cp -c): no extra disk until one side changes. +# tokensynth's setup on a Mac or Linux: setup.py beside this (what it installs, where, how big), run through uv as raea runs it +# on the model's first use (raea_io_assets::run_setup). On Windows: +# uv run --no-project --python 3.11 python -I tools\tokensynth\setup.py +# uv: $UV, the one on the PATH, raea's own (~/.cache/raea-tools/uv), else on an Apple-silicon Mac tools/ensure.sh +# fetches that one now. # tools/tokensynth/setup.sh set -e -DIR=${RAEA_TOKENSYNTH:-$HOME/.cache/raea-tokensynth} -HERE=$(cd "$(dirname "$0")" && pwd) -FROM=${RAEA_TOKENSYNTH_FROM:-$HOME/.cache/raea-tokensynth} -mkdir -p "$DIR/weights" "$DIR/hf" -DIR=$(cd "$DIR" && pwd) -[ "$(cd "$FROM" 2>/dev/null && pwd)" = "$DIR" ] && FROM= -# one setup at a time: a render and a CLI asking for the first note together would both start one -if ! mkdir "$DIR/.setup-lock" 2>/dev/null; then - echo "tokensynth setup: another setup is running in $DIR (remove $DIR/.setup-lock if it isn't)" >&2 - exit 75 -fi -trap 'rm -rf "$DIR/.setup-lock"' EXIT -trap 'exit 130' INT TERM -cd "$DIR" -say() { echo "tokensynth setup: $*" >&2; } -clone() { cp -cR "$1" "$2" 2>/dev/null || cp -R "$1" "$2"; } - -if [ ! -d repo ]; then - if [ -n "$FROM" ] && [ -d "$FROM/repo" ]; then clone "$FROM/repo" repo; else git clone -q --depth 1 https://github.com/KyungsuKim42/tokensynth repo; fi -fi -git -C repo rev-parse HEAD > repo.commit -# the versions the spike ran (2026-10-18): laion-clap 1.1.6 needs transformers 4.30; scipy 1.15's wheel won't load here -# coremltools for convert.py's Core ML models (run.py --precision coreml). An existing venv that has them is left alone -if ! .venv/bin/python -c "import torch, laion_clap, dac, tokensynth, coremltools" 2>/dev/null; then - say "making the venv (about 1.1 GB)" - [ -x .venv/bin/python ] || uv venv -q --python 3.11 .venv - VIRTUAL_ENV="$DIR/.venv" uv pip install -q --python .venv/bin/python "torch==2.5.1" "torchaudio==2.5.1" "transformers==4.30.2" "scipy==1.14.1" "coremltools==8.3.0" laion-clap descript-audio-codec pretty_midi soundfile numpy -e repo -fi -for f in token_synth_aug.pt token_synth_unconditional.pt dac_weights_44khz_8kbps_0.0.1.pt clap_music_audioset_epoch_15_esc_90.14.pt; do - if [ ! -s "weights/$f" ]; then - if [ -n "$FROM" ] && [ -s "$FROM/weights/$f" ]; then - clone "$FROM/weights/$f" "weights/$f.part" - else - say "downloading weights/$f" - curl -fL --retry 3 --progress-bar -o "weights/$f.part" "https://huggingface.co/KyungsuKim/TokenSynth/resolve/main/$f" - fi - mv -f "weights/$f.part" "weights/$f" - fi -done -# the converted runtime from the other install, when it has it whole: convert.py then finds everything made -for d in weights-fp16 coreml; do - if [ ! -e "$d" ] && [ -n "$FROM" ] && [ -d "$FROM/$d" ]; then clone "$FROM/$d" "$d"; fi -done -cp "$HERE/run.py" "$DIR/run.py" -# the smaller runtime run.py's other precisions read (weights-fp16/, coreml/); keeps what is already made -nice -n 15 .venv/bin/python -I "$HERE/convert.py" --dir "$DIR" -date -u +%Y-%m-%dT%H:%M:%SZ > "$DIR/.ready" -du -sh "$DIR/weights" "$DIR/weights-fp16" "$DIR/coreml" "$DIR/.venv" >&2 -say "ready in $DIR (repo $(cat repo.commit))" +here=$(cd "$(dirname "$0")" && pwd) +own=${RAEA_TOOLS_CACHE:-$HOME/.cache/raea-tools}/uv/uv +UV=${UV:-$(command -v uv || true)} +[ -n "$UV" ] || [ ! -x "$own" ] || UV=$own +[ -n "$UV" ] || [ "$(uname -sm)" != "Darwin arm64" ] || UV=$("$here/../ensure.sh" uv) +[ -n "$UV" ] || { echo "tokensynth setup: needs uv: install it (https://docs.astral.sh/uv), or let raea fetch its own on the model's first use" >&2; exit 1; } +export UV +exec "$UV" run -q --no-project --python 3.11 python -I "$here/setup.py" "$@"