From 27a5a26744b868757d1ddfbcf22c5b011b57566b Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Fri, 13 Nov 2026 11:30:39 -0800 Subject: [PATCH 1/3] A render starts in a fraction of the time in a real project: the export cache and the bundler no longer read or stat every file under the edit's folder --- runtime/bundle/src/cache.rs | 22 +++++++++++++++++----- runtime/bundle/src/rewrite.rs | 14 +++++++++++++- runtime/bundle/src/tests.rs | 14 ++++++++++++++ runtime/export/src/render.rs | 2 ++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/runtime/bundle/src/cache.rs b/runtime/bundle/src/cache.rs index f8d024cd..00371c5f 100644 --- a/runtime/bundle/src/cache.rs +++ b/runtime/bundle/src/cache.rs @@ -89,7 +89,11 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio // never a stale export) let text = String::from_utf8_lossy(&bundle); let made: std::collections::HashSet = std::fs::read_to_string(root.join("outputs.list")).unwrap_or_default().lines().map(PathBuf::from).collect(); - files.retain(|p| !made.contains(&std::fs::canonicalize(p).unwrap_or_else(|_| p.clone())) || p.file_stem().is_some_and(|n| text.contains(&*n.to_string_lossy()))); + // (a file is one of them only if its name is one of theirs, or it is a link to one: only those are resolved, where + // every file under the edit's folder was, thousands of them before a render's first frame) + let names: std::collections::HashSet<&std::ffi::OsStr> = made.iter().filter_map(|m| m.file_name()).collect(); + let may_be_made = |p: &Path| p.file_name().is_some_and(|n| names.contains(n)) || p.symlink_metadata().is_ok_and(|m| m.file_type().is_symlink()); + files.retain(|p| !may_be_made(p) || !made.contains(&std::fs::canonicalize(p).unwrap_or_else(|_| p.clone())) || p.file_stem().is_some_and(|n| text.contains(&*n.to_string_lossy()))); // absolute media paths the bundle names (an import from outside the edit's folder becomes its path) for q in text.split('"').skip(1).step_by(2) { if q.starts_with('/') && q.len() < 4096 && Path::new(q).extension().and_then(|x| x.to_str()).is_some_and(|x| MEDIA_EXTS.contains(&x.to_ascii_lowercase().as_str())) { @@ -99,6 +103,11 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio files.push(std::env::current_exe().ok().and_then(|e| std::fs::canonicalize(e).ok())?); files.sort(); files.dedup(); + // the optimized copies there are, listed once. Asking each media file for its own (raea_io_media::mezz::has and + // ranges) read the file's first and last MiB and listed this folder again, for every picture, sound and video + // under the edit's folder: 492 files in one real project and 2,417 in another, before a render's first frame. + // Only a video has copies, and with none in the folder no file has one + let copies: Vec = std::fs::read_dir(raea_io_media::mezz::dir()).map(|d| d.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect()).unwrap_or_default(); for p in &files { eat(p.as_os_str().as_encoded_bytes()); match std::fs::metadata(p) { @@ -110,13 +119,16 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio } // which file its decoders read: a video's optimized copy, once complete, changes the pixels without touching // the source (raea_io_media::mezz) - if MEDIA_EXTS.contains(&p.extension().and_then(|x| x.to_str()).unwrap_or("").to_ascii_lowercase().as_str()) { - if raea_io_media::mezz::has(p) { + let video = matches!(p.extension().and_then(|x| x.to_str()).unwrap_or("").to_ascii_lowercase().as_str(), "mp4" | "mov" | "m4v"); + if video && !copies.is_empty() && let Some(fp) = raea_io_media::mezz::fingerprint(p) { + if copies.iter().any(|n| n.strip_prefix(fp.as_str()).is_some_and(|rest| rest.starts_with('.') && !rest.contains("partial"))) { eat(b"mezz"); } // its range copies too: the frames inside them come from the copies - for r in raea_io_media::mezz::ranges(p) { - eat(format!("mezz@{}-{}", r.first, r.last).as_bytes()); + if copies.iter().any(|n| n.strip_prefix(fp.as_str()).is_some_and(|rest| rest.starts_with('@'))) { + for r in raea_io_media::mezz::ranges(p) { + eat(format!("mezz@{}-{}", r.first, r.last).as_bytes()); + } } } } diff --git a/runtime/bundle/src/rewrite.rs b/runtime/bundle/src/rewrite.rs index 910e2e8e..932dadcd 100644 --- a/runtime/bundle/src/rewrite.rs +++ b/runtime/bundle/src/rewrite.rs @@ -126,8 +126,20 @@ pub fn sources(root: &Path) -> R> { let mut stack = vec![root.to_path_buf()]; while let Some(d) = stack.pop() { for e in std::fs::read_dir(&d).map_err(|e| format!("{}: {e}", d.display()))?.flatten() { + // what copy_dir leaves out is no source of the edit's: a project's .git, node_modules and target were + // walked whole here, each entry asked what it is (2 s before a render's first frame in a project of ten + // thousand files, bundle cached or not) + let name = e.file_name(); + if name.to_string_lossy().starts_with('.') || name == "node_modules" || name == "target" { + continue; + } let p = e.path(); - if p.is_dir() { + // the entry's own kind, without another stat; a link is followed + let dir = match e.file_type() { + Ok(t) if !t.is_symlink() => t.is_dir(), + _ => p.is_dir(), + }; + if dir { stack.push(p); } else if matches!(p.extension().and_then(|x| x.to_str()), Some("ts" | "tsx" | "js" | "jsx" | "mjs")) { out.push(p.strip_prefix(root).unwrap().to_path_buf()); diff --git a/runtime/bundle/src/tests.rs b/runtime/bundle/src/tests.rs index 49399225..d3f47620 100644 --- a/runtime/bundle/src/tests.rs +++ b/runtime/bundle/src/tests.rs @@ -89,3 +89,17 @@ fn a_project_s_own_tsconfig_is_not_in_the_work_copy() { assert!(d.join("from/tsconfig.json").exists()); let _ = std::fs::remove_dir_all(&d); } + +#[test] +fn an_edit_s_sources_leave_out_hidden_folders_and_packages() { + let d = std::env::temp_dir().join(format!("raea-bundle-sources-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + for f in ["a.tsx", "parts/b.ts", ".git/hooks/c.js", "node_modules/pkg/d.js", "target/e.ts", "out/f.json"] { + std::fs::create_dir_all(d.join(f).parent().unwrap()).unwrap(); + std::fs::write(d.join(f), "").unwrap(); + } + let mut got = sources(&d).unwrap(); + got.sort(); + assert_eq!(got, vec![PathBuf::from("a.tsx"), PathBuf::from("parts/b.ts")]); + let _ = std::fs::remove_dir_all(&d); +} diff --git a/runtime/export/src/render.rs b/runtime/export/src/render.rs index a4b36e5d..dfb65c64 100644 --- a/runtime/export/src/render.rs +++ b/runtime/export/src/render.rs @@ -55,6 +55,7 @@ pub fn render(args: &[String]) -> R { unsafe { std::env::set_var("RAEA_SOURCE_STAMPS", "1") }; } bundle_composition(&tsx, &js, &o, None)?; + raea_runtime_react::startup_mark("the composition bundled"); let wanted = match (flag(args, "--frame"), flag(args, "--at")) { (Some(f), _) => Some(vec![f.parse().map_err(|_| "--frame takes a frame number")?]), (_, Some(at)) => Some(at.split(',').map(|f| f.trim().parse::().map_err(|_| "--at takes frame numbers".to_string())).collect::>()?), @@ -91,6 +92,7 @@ pub fn render(args: &[String]) -> R { }; // a whole video made from these same inputs before is the export: cloned from the cache, not rendered again let cache = if wanted.is_none() && !raea_sys_layers::on() { export_cache(&tsx, &js, &out, args) } else { None }; + raea_runtime_react::startup_mark("the export cache looked up"); let manifest = MANIFEST.lock().unwrap().clone(); if let Some(c) = &cache { // with --manifest, the manifest kept beside the export comes back too From ae9e7eab31b638658a1025ade0cdbf6ff2164ec7 Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Fri, 13 Nov 2026 11:32:46 -0800 Subject: [PATCH 2/3] bundle: the copies' lookup is one closure, shorter notes (dupes, size) --- runtime/bundle/src/cache.rs | 20 ++++++++------------ runtime/bundle/src/rewrite.rs | 5 ++--- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/runtime/bundle/src/cache.rs b/runtime/bundle/src/cache.rs index 00371c5f..a666c5a1 100644 --- a/runtime/bundle/src/cache.rs +++ b/runtime/bundle/src/cache.rs @@ -89,8 +89,7 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio // never a stale export) let text = String::from_utf8_lossy(&bundle); let made: std::collections::HashSet = std::fs::read_to_string(root.join("outputs.list")).unwrap_or_default().lines().map(PathBuf::from).collect(); - // (a file is one of them only if its name is one of theirs, or it is a link to one: only those are resolved, where - // every file under the edit's folder was, thousands of them before a render's first frame) + // (a file is one of them only if its name is one of theirs, or it is a link to one: only those are resolved) let names: std::collections::HashSet<&std::ffi::OsStr> = made.iter().filter_map(|m| m.file_name()).collect(); let may_be_made = |p: &Path| p.file_name().is_some_and(|n| names.contains(n)) || p.symlink_metadata().is_ok_and(|m| m.file_type().is_symlink()); files.retain(|p| !may_be_made(p) || !made.contains(&std::fs::canonicalize(p).unwrap_or_else(|_| p.clone())) || p.file_stem().is_some_and(|n| text.contains(&*n.to_string_lossy()))); @@ -103,10 +102,8 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio files.push(std::env::current_exe().ok().and_then(|e| std::fs::canonicalize(e).ok())?); files.sort(); files.dedup(); - // the optimized copies there are, listed once. Asking each media file for its own (raea_io_media::mezz::has and - // ranges) read the file's first and last MiB and listed this folder again, for every picture, sound and video - // under the edit's folder: 492 files in one real project and 2,417 in another, before a render's first frame. - // Only a video has copies, and with none in the folder no file has one + // the optimized copies there are, listed once: asking each media file for its own read its first and last MiB and + // listed this folder again (2,417 files in one real project, before a render's first frame). Only a video has any let copies: Vec = std::fs::read_dir(raea_io_media::mezz::dir()).map(|d| d.flatten().map(|e| e.file_name().to_string_lossy().into_owned()).collect()).unwrap_or_default(); for p in &files { eat(p.as_os_str().as_encoded_bytes()); @@ -121,14 +118,13 @@ pub fn export_cache(tsx: &Path, js: &Path, out: &Path, args: &[String]) -> Optio // the source (raea_io_media::mezz) let video = matches!(p.extension().and_then(|x| x.to_str()).unwrap_or("").to_ascii_lowercase().as_str(), "mp4" | "mov" | "m4v"); if video && !copies.is_empty() && let Some(fp) = raea_io_media::mezz::fingerprint(p) { - if copies.iter().any(|n| n.strip_prefix(fp.as_str()).is_some_and(|rest| rest.starts_with('.') && !rest.contains("partial"))) { + // its whole copy (FP.EXT) and its range copies (FP@A-B.EXT): the frames inside them come from the copies + let named = |mark: char| copies.iter().any(|n| n.strip_prefix(fp.as_str()).is_some_and(|rest| rest.starts_with(mark) && !rest.contains("partial"))); + if named('.') { eat(b"mezz"); } - // its range copies too: the frames inside them come from the copies - if copies.iter().any(|n| n.strip_prefix(fp.as_str()).is_some_and(|rest| rest.starts_with('@'))) { - for r in raea_io_media::mezz::ranges(p) { - eat(format!("mezz@{}-{}", r.first, r.last).as_bytes()); - } + for r in named('@').then(|| raea_io_media::mezz::ranges(p)).unwrap_or_default() { + eat(format!("mezz@{}-{}", r.first, r.last).as_bytes()); } } } diff --git a/runtime/bundle/src/rewrite.rs b/runtime/bundle/src/rewrite.rs index 932dadcd..5420371c 100644 --- a/runtime/bundle/src/rewrite.rs +++ b/runtime/bundle/src/rewrite.rs @@ -126,9 +126,8 @@ pub fn sources(root: &Path) -> R> { let mut stack = vec![root.to_path_buf()]; while let Some(d) = stack.pop() { for e in std::fs::read_dir(&d).map_err(|e| format!("{}: {e}", d.display()))?.flatten() { - // what copy_dir leaves out is no source of the edit's: a project's .git, node_modules and target were - // walked whole here, each entry asked what it is (2 s before a render's first frame in a project of ten - // thousand files, bundle cached or not) + // what copy_dir leaves out is no source of the edit's (a project's .git and node_modules were walked whole: + // 2 s before a render's first frame in a project of ten thousand files) let name = e.file_name(); if name.to_string_lossy().starts_with('.') || name == "node_modules" || name == "target" { continue; From 20501b17dc965069627b0ba7c6b898ac68ac48ac Mon Sep 17 00:00:00 2001 From: shinyobjectz Date: Fri, 13 Nov 2026 11:38:39 -0800 Subject: [PATCH 3/3] RAEA_MODELS_TRACE=1: a wait for a scene's models says how many updates it ran and for which models --- runtime/scene3d/src/model.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/runtime/scene3d/src/model.rs b/runtime/scene3d/src/model.rs index 489bef24..9b9ad336 100644 --- a/runtime/scene3d/src/model.rs +++ b/runtime/scene3d/src/model.rs @@ -340,9 +340,13 @@ pub fn await_models(_app: &mut App) -> Result<(), String> { #[cfg(not(target_arch = "wasm32"))] pub fn await_models(app: &mut App) -> Result<(), String> { let mut t0 = bevy::platform::time::Instant::now(); + let t_all = t0; let mut most = 0usize; // the cameras off while models load, so the draws are binned once, whole (#48): raea-sys-post's views.rs let mut off: Option> = None; + // RAEA_MODELS_TRACE=1: each wait that ran an update says how many and for which models (a render's frame time) + let trace = std::env::var("RAEA_MODELS_TRACE").as_deref() == Ok("1"); + let mut pumped = 0usize; loop { let world = app.world_mut(); if let Some(e) = world.resource_mut::().0.take() { @@ -367,8 +371,17 @@ pub fn await_models(app: &mut App) -> Result<(), String> { } if waiting == 0 { if let Some(cams) = off { raea_sys_post::views::on(app, cams); } + if trace && pumped > 0 { + eprintln!("raea models: waited {pumped} update(s), {:.1} ms", t_all.elapsed().as_secs_f64() * 1e3); + } return Ok(()); } + if trace && pumped == 0 { + let mut q = app.world_mut().query::<&Model3d>(); + let who: Vec = q.iter(app.world()).filter(|m| !(matches!(m.state, ModelState::Ready { .. }) && m.settled())).map(|m| format!("{} ({}; surfaces {}, toon {}, shaders {})", m.id, match m.state { ModelState::Loading => "loading", ModelState::Spawned(_) => "spawned", ModelState::Ready { .. } => "ready" }, m.surfaces.len(), m.toon.len(), m.shaders.len())).take(4).collect(); + eprintln!("raea models: waiting on {waiting}: {}", who.join("; ")); + } + pumped += 1; if done > most { most = done; t0 = bevy::platform::time::Instant::now();