From 95d117475fc6105654e7d23bff789fea1d1762a2 Mon Sep 17 00:00:00 2001 From: midagedev Date: Fri, 25 Sep 2026 08:53:47 +0900 Subject: [PATCH 1/5] fix(cargo-oxide): install the cached backend by rename, not in place install_backend_into copied the new backend over the cached .so with fs::copy. That truncates and rewrites the same inode, so a rustc that already loaded the old file can die with SIGBUS. Copy to a temporary file in the cache dir, then rename it over the .so. If the copy or the rename fails, remove the temporary file and return the error. Tests: an open handle keeps reading the old bytes and inode after an install; a failed copy or rename leaves the old .so and no temporary file. Signed-off-by: midagedev --- cuda-oxide/crates/cargo-oxide/src/backend.rs | 139 ++++++++++++++++++- 1 file changed, 138 insertions(+), 1 deletion(-) diff --git a/cuda-oxide/crates/cargo-oxide/src/backend.rs b/cuda-oxide/crates/cargo-oxide/src/backend.rs index 321e0b5aaa..603b008b82 100644 --- a/cuda-oxide/crates/cargo-oxide/src/backend.rs +++ b/cuda-oxide/crates/cargo-oxide/src/backend.rs @@ -1277,6 +1277,10 @@ fn auto_fetch_and_build() -> PathBuf { /// Copies a freshly built backend into `cache_dir` and records the toolchain /// fingerprint and the source commit beside it. /// +/// The `.so` is copied to a temporary file and renamed over the old one, so a +/// `rustc` that already mapped the old file keeps reading it. On error the +/// temporary file is removed and the old `.so` stays. +/// /// The fingerprint must be written whenever the `.so` is. A `.so` installed /// without one falls back to the mtime checks, which cannot see a toolchain /// swap, so the next lookup would load a backend linked against the wrong @@ -1300,12 +1304,29 @@ fn install_backend_into( // the only intermediate state is "no record", which counts as stale: a // copy interrupted halfway can never be handed out under the old commit. record_source_rev(cache_dir, None); - std::fs::copy(built_so, &so_path)?; + let temporary = temporary_backend_path(cache_dir); + if let Err(error) = + std::fs::copy(built_so, &temporary).and_then(|_| std::fs::rename(&temporary, &so_path)) + { + let _ = std::fs::remove_file(&temporary); + return Err(error); + } write_toolchain_fingerprint(cache_dir, build_dir); record_source_rev(cache_dir, source_rev); Ok(so_path) } +/// A temporary name beside the `.so` (same filesystem, so the rename is +/// atomic). The pid and a random part keep two installers apart. +fn temporary_backend_path(cache_dir: &Path) -> PathBuf { + use std::hash::{BuildHasher, RandomState}; + let nonce = RandomState::new().hash_one(()); + cache_dir.join(format!( + "librustc_codegen_cuda.so.{}.{nonce:016x}.tmp", + std::process::id() + )) +} + /// What [`publish_to_cache`] installed. pub struct PublishedBackend { /// The cached `.so`. @@ -1867,6 +1888,106 @@ mod tests { ); } + /// The new backend is shorter, so an in-place rewrite would also truncate + /// the file under the open handle. + #[cfg(unix)] + #[test] + fn installing_leaves_a_loaded_backend_intact() { + use std::io::{Read, Seek}; + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + + const OLD: &str = "old backend, longer than the new one"; + const NEW: &str = "new backend"; + let dir = tempdir(); + let cache = dir.join("cache"); + let old = dir.join("old.so"); + let new = dir.join("new.so"); + std::fs::write(&old, OLD).unwrap(); + std::fs::write(&new, NEW).unwrap(); + std::fs::set_permissions(&old, std::fs::Permissions::from_mode(0o644)).unwrap(); + std::fs::set_permissions(&new, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let installed = + install_backend_into(&cache, &old, &dir, None).expect("install must succeed"); + let mut loaded = std::fs::File::open(&installed).unwrap(); + let mut seen = String::new(); + loaded.read_to_string(&mut seen).unwrap(); + assert_eq!(seen, OLD); + + install_backend_into(&cache, &new, &dir, None).expect("install must succeed"); + + seen.clear(); + loaded.rewind().unwrap(); + loaded.read_to_string(&mut seen).unwrap(); + assert_eq!( + seen, OLD, + "a handle opened before the install must keep reading the old backend" + ); + assert_eq!( + std::fs::read_to_string(&installed).unwrap(), + NEW, + "the path must yield the new backend" + ); + let metadata = std::fs::metadata(&installed).unwrap(); + assert_ne!( + metadata.ino(), + loaded.metadata().unwrap().ino(), + "the new backend must be a new file, not the old inode rewritten" + ); + assert_eq!( + metadata.permissions().mode() & 0o777, + 0o755, + "the installed backend must take the built file's mode" + ); + assert_eq!( + unexpected_cache_entries(&cache), + Vec::::new(), + "installing must not leave a temporary file in the cache" + ); + } + + #[test] + fn a_failed_install_keeps_the_cached_backend() { + let dir = tempdir(); + let cache = dir.join("cache"); + let source = dir.join("built.so"); + std::fs::write(&source, b"built").unwrap(); + let installed = + install_backend_into(&cache, &source, &dir, None).expect("install must succeed"); + + let error = install_backend_into(&cache, &dir.join("missing.so"), &dir, None) + .expect_err("installing a build that does not exist must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound); + assert_eq!( + std::fs::read(&installed).unwrap(), + b"built", + "a failed install must keep the cached backend" + ); + assert_eq!( + unexpected_cache_entries(&cache), + Vec::::new(), + "a failed install must not leave a temporary file in the cache" + ); + } + + /// The backend's name is taken by a directory, so the rename fails. + #[test] + fn a_failed_rename_removes_the_temporary_copy() { + let dir = tempdir(); + let cache = dir.join("cache"); + std::fs::create_dir_all(cache.join("librustc_codegen_cuda.so").join("occupied")).unwrap(); + let source = dir.join("built.so"); + std::fs::write(&source, b"built").unwrap(); + + install_backend_into(&cache, &source, &dir, None) + .expect_err("installing over a directory must fail"); + assert_eq!( + unexpected_cache_entries(&cache), + Vec::::new(), + "a failed install must not leave its temporary copy in the cache" + ); + } + /// A cached `.so` whose recorded toolchain fingerprint differs from the /// active toolchain must be `StaleVsToolchain`, even when the mtimes alone /// would call it fresh. This is the case the mtime checks miss: the active @@ -2579,4 +2700,20 @@ mod tests { f.write_all(contents).unwrap(); f.set_modified(mtime).unwrap(); } + + /// Names in `cache` other than the backend and the two records beside it. + fn unexpected_cache_entries(cache: &Path) -> Vec { + let known = [ + "librustc_codegen_cuda.so", + TOOLCHAIN_FINGERPRINT_FILE, + SOURCE_REV_FILE, + ]; + let mut names = std::fs::read_dir(cache) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .filter(|name| !known.contains(&name.as_str())) + .collect::>(); + names.sort(); + names + } } From b9243f59c19e9f5b435547f4f53313266f5576cf Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:38:37 +0530 Subject: [PATCH 2/5] fix(cargo-oxide): clear the commit record just before the rename install_backend_into removed source-rev.txt before copying the new backend to its temporary. The copy no longer touches the cached .so, so a copy that fails, for example on a full disk, left the old backend in place without its record, and the next pinned lookup rebuilt a cache that was still valid. The record is now removed between the copy and the rename, so the only state without a record is the short window from the rename to the new record, and a failed copy costs nothing. The failed-copy test installs under a commit and checks that the record survives the failed second install; it fails with the old ordering. Signed-off-by: nihalpasham --- cuda-oxide/crates/cargo-oxide/src/backend.rs | 29 ++++++++++++++------ 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/cuda-oxide/crates/cargo-oxide/src/backend.rs b/cuda-oxide/crates/cargo-oxide/src/backend.rs index 4009fc6fea..43b66e08ba 100644 --- a/cuda-oxide/crates/cargo-oxide/src/backend.rs +++ b/cuda-oxide/crates/cargo-oxide/src/backend.rs @@ -1300,14 +1300,18 @@ fn install_backend_into( ) -> std::io::Result { std::fs::create_dir_all(cache_dir)?; let so_path = cache_dir.join("librustc_codegen_cuda.so"); - // Drop the old record before the copy and write the new one after it, so - // the only intermediate state is "no record", which counts as stale: a - // copy interrupted halfway can never be handed out under the old commit. - record_source_rev(cache_dir, None); let temporary = temporary_backend_path(cache_dir); - if let Err(error) = - std::fs::copy(built_so, &temporary).and_then(|_| std::fs::rename(&temporary, &so_path)) - { + if let Err(error) = std::fs::copy(built_so, &temporary) { + let _ = std::fs::remove_file(&temporary); + return Err(error); + } + // The copy leaves the old `.so` and its record untouched, so a copy that + // fails costs nothing. Drop the old record just before the rename and + // write the new one after it, so the only intermediate state is "no + // record", which counts as stale: the new `.so` is never handed out under + // the old commit. + record_source_rev(cache_dir, None); + if let Err(error) = std::fs::rename(&temporary, &so_path) { let _ = std::fs::remove_file(&temporary); return Err(error); } @@ -1946,14 +1950,16 @@ mod tests { ); } + /// The copy fails before anything in the cache changes, so the backend + /// and the commit recorded for it both stay. #[test] fn a_failed_install_keeps_the_cached_backend() { let dir = tempdir(); let cache = dir.join("cache"); let source = dir.join("built.so"); std::fs::write(&source, b"built").unwrap(); - let installed = - install_backend_into(&cache, &source, &dir, None).expect("install must succeed"); + let installed = install_backend_into(&cache, &source, &dir, Some("aaaa1111")) + .expect("install must succeed"); let error = install_backend_into(&cache, &dir.join("missing.so"), &dir, None) .expect_err("installing a build that does not exist must fail"); @@ -1963,6 +1969,11 @@ mod tests { b"built", "a failed install must keep the cached backend" ); + assert_eq!( + recorded_source_rev(&cache).as_deref(), + Some("aaaa1111"), + "a failed copy must keep the record of the backend that is still there" + ); assert_eq!( unexpected_cache_entries(&cache), Vec::::new(), From ab7e6d84b55f783a2722537017ecee820cf02aaf Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:38:37 +0530 Subject: [PATCH 3/5] fix(cargo-oxide): remove stranded backend temporaries on update An install killed between its copy and its rename leaves librustc_codegen_cuda.so...tmp in the cache directory, as large as the backend, and nothing removed it. cargo oxide update now removes such temporaries after clearing the cache. The sweep runs only on that explicit path: clear_cache_contents doubles as the automatic invalidation any project's lookup can run, and an install cannot tell a stranded temporary from another installer's live one, so neither of them removes temporaries. is_backend_temporary sits beside temporary_backend_path so the name policy lives in one place. Tests cover the sweep, an install that leaves a planted temporary alone, and the automatic path that leaves it alone. Signed-off-by: nihalpasham --- cuda-oxide/crates/cargo-oxide/src/backend.rs | 98 +++++++++++++++++++- 1 file changed, 96 insertions(+), 2 deletions(-) diff --git a/cuda-oxide/crates/cargo-oxide/src/backend.rs b/cuda-oxide/crates/cargo-oxide/src/backend.rs index 43b66e08ba..1419276853 100644 --- a/cuda-oxide/crates/cargo-oxide/src/backend.rs +++ b/cuda-oxide/crates/cargo-oxide/src/backend.rs @@ -875,7 +875,8 @@ fn invalidate_cache(cache_dir: &Path) { clear_cache_contents(cache_dir); } -/// Drop the cached `.so` and source tree used by external projects. +/// Drop the cached `.so` and source tree used by external projects, and any +/// temporary an interrupted install left beside them. /// /// Used by `cargo oxide update` for an explicit refresh, distinct from the /// automatic upgrade path in [`invalidate_cache`]. @@ -886,9 +887,27 @@ pub fn clear_cached_backend() -> Option { cache_dir.display() ); clear_cache_contents(&cache_dir); + remove_stranded_backend_temporaries(&cache_dir); Some(cache_dir) } +/// Removes the temporaries left by installs killed between their copy and +/// their rename (the names [`temporary_backend_path`] produces); each one is +/// as large as the backend. Only the explicit `cargo oxide update` path +/// removes them. The automatic invalidation in [`invalidate_cache`] runs from +/// any project's lookup, possibly beside another project's installer, whose +/// live temporary must stay until its rename. +fn remove_stranded_backend_temporaries(cache_dir: &Path) { + let Ok(entries) = std::fs::read_dir(cache_dir) else { + return; + }; + for entry in entries.flatten() { + if is_backend_temporary(&entry.file_name().to_string_lossy()) { + let _ = std::fs::remove_file(entry.path()); + } + } +} + fn clear_cache_contents(cache_dir: &Path) { let _ = std::fs::remove_file(cache_dir.join("librustc_codegen_cuda.so")); let _ = std::fs::remove_dir_all(cache_dir.join("src")); @@ -1279,7 +1298,9 @@ fn auto_fetch_and_build() -> PathBuf { /// /// The `.so` is copied to a temporary file and renamed over the old one, so a /// `rustc` that already mapped the old file keeps reading it. On error the -/// temporary file is removed and the old `.so` stays. +/// temporary file is removed and the old `.so` stays. A process killed between +/// the copy and the rename leaves the temporary behind; `cargo oxide update` +/// removes it (see [`remove_stranded_backend_temporaries`]). /// /// The fingerprint must be written whenever the `.so` is. A `.so` installed /// without one falls back to the mtime checks, which cannot see a toolchain @@ -1331,6 +1352,11 @@ fn temporary_backend_path(cache_dir: &Path) -> PathBuf { )) } +/// True for the names [`temporary_backend_path`] produces. +fn is_backend_temporary(name: &str) -> bool { + name.starts_with("librustc_codegen_cuda.so.") && name.ends_with(".tmp") +} + /// What [`publish_to_cache`] installed. pub struct PublishedBackend { /// The cached `.so`. @@ -1604,6 +1630,24 @@ mod tests { ); } + /// `clear_cache_contents` doubles as the automatic invalidation that any + /// project's lookup can run, possibly beside another project's installer, + /// so it leaves that installer's temporary alone. Only `cargo oxide + /// update` removes temporaries. + #[test] + fn clear_cache_contents_leaves_a_live_temporary_alone() { + let dir = tempdir(); + let live = temporary_backend_path(&dir); + std::fs::write(&live, b"half-copied").unwrap(); + + clear_cache_contents(&dir); + + assert!( + live.exists(), + "an automatic invalidation must not remove another installer's temporary" + ); + } + /// A backend source input newer than the cached `.so` must report /// `StaleVsSource` (the "developer advanced the source" case that issue /// #49's binary check alone misses). To isolate the source signal from @@ -1981,6 +2025,56 @@ mod tests { ); } + /// A process killed between the copy and the rename leaves its temporary + /// behind. A later install leaves it alone, since it cannot tell it from + /// another installer's live copy; `cargo oxide update` removes it. + #[test] + fn update_removes_a_temporary_left_by_a_killed_install() { + let dir = tempdir(); + let cache = dir.join("cache"); + let source = dir.join("built.so"); + std::fs::write(&source, b"built").unwrap(); + let installed = install_backend_into(&cache, &source, &dir, Some("aaaa1111")) + .expect("install must succeed"); + // One named by a process that is gone and one named by this process, + // so the sweep and the name policy are checked against each other. + let stranded = [ + cache.join("librustc_codegen_cuda.so.1.00000000deadbeef.tmp"), + temporary_backend_path(&cache), + ]; + for path in &stranded { + std::fs::write(path, b"half-copied").unwrap(); + } + + install_backend_into(&cache, &source, &dir, Some("aaaa1111")) + .expect("install must succeed"); + for path in &stranded { + assert!( + path.exists(), + "an install must not remove what may be another installer's temporary" + ); + } + + remove_stranded_backend_temporaries(&cache); + for path in &stranded { + assert!( + !path.exists(), + "`cargo oxide update` must remove a stranded temporary" + ); + } + assert_eq!( + std::fs::read(&installed).unwrap(), + b"built", + "the sweep must leave the backend alone" + ); + assert_eq!( + recorded_source_rev(&cache).as_deref(), + Some("aaaa1111"), + "the sweep must leave the record alone" + ); + assert_eq!(unexpected_cache_entries(&cache), Vec::::new()); + } + /// The backend's name is taken by a directory, so the rename fails. #[test] fn a_failed_rename_removes_the_temporary_copy() { From 48c26d12c0351cfa6b40c3cfdfd1df7710fc1056 Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:38:37 +0530 Subject: [PATCH 4/5] test(cargo-oxide): close the loaded handle before listing the cache installing_leaves_a_loaded_backend_intact listed the cache directory while its handle to the replaced backend was still open. A network filesystem keeps a replaced file under a hidden name until the last handle closes, which would fail the "no temporary left" assertion on such a mount. The handle is closed first, so the assertion depends only on the install. Signed-off-by: nihalpasham --- cuda-oxide/crates/cargo-oxide/src/backend.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cuda-oxide/crates/cargo-oxide/src/backend.rs b/cuda-oxide/crates/cargo-oxide/src/backend.rs index 1419276853..2e5ce0d5ff 100644 --- a/cuda-oxide/crates/cargo-oxide/src/backend.rs +++ b/cuda-oxide/crates/cargo-oxide/src/backend.rs @@ -1987,6 +1987,9 @@ mod tests { 0o755, "the installed backend must take the built file's mode" ); + // Closed first: a network filesystem keeps a replaced file under a + // hidden name while a handle to it is open. + drop(loaded); assert_eq!( unexpected_cache_entries(&cache), Vec::::new(), From 4b4558e7467e6ee7b6d38400ae0b71fdb86d8cdc Mon Sep 17 00:00:00 2001 From: nihalpasham Date: Mon, 19 Oct 2026 15:39:29 +0530 Subject: [PATCH 5/5] docs(book): show how the cached backend is installed A one-page figure for the book and the pull request, in the style of the tile-libraries direct-linkage figure. Signed-off-by: nihalpasham --- .../images/backend-install-by-rename.svg | 108 ++++++++++++++++++ .../getting-started/installation.md | 9 ++ 2 files changed, 117 insertions(+) create mode 100644 cuda-oxide/cuda-oxide-book/getting-started/images/backend-install-by-rename.svg diff --git a/cuda-oxide/cuda-oxide-book/getting-started/images/backend-install-by-rename.svg b/cuda-oxide/cuda-oxide-book/getting-started/images/backend-install-by-rename.svg new file mode 100644 index 0000000000..22f5473993 --- /dev/null +++ b/cuda-oxide/cuda-oxide-book/getting-started/images/backend-install-by-rename.svg @@ -0,0 +1,108 @@ + + + + + +Installing the cached backend by rename, not in place +Every rustc that compiles a kernel maps the shared 428 MB librustc_codegen_cuda.so. std::fs::copy rewrote that inode under them; a sibling temporary plus rename leaves it alone. +THE SETUP (one file, many readers, three writers) +BEFORE (fs::copy rewrites the mapped inode) +AFTER (copy beside it, rename over the name) + + + +Who installs the backend +cargo oxide setup publish_to_cache +a standalone project's rebuild build_and_cache +clone of main, no dependency auto_fetch_and_build +all three call install_backend_into, at any time + +install_backend_into(cache_dir, built_so, ..) + +~/.cargo/cuda-oxide/ +librustc_codegen_cuda.so 428 MB, inode A +source-rev.txt the commit it came from +toolchain-fingerprint.txt the rustc that built it + +dlopen at start, mapped MAP_PRIVATE until exit + +every rustc that compiles a kernel +rustc -Zcodegen-backend=<the cached .so> ... +holds the mapping for the whole compile; ld.so's +relocations live in private pages of that inode +A reinstall can land while such a rustc runs: +setup in one shell, a kernel build in another. + +std::fs::copy(built_so, &so_path) +opens the destination with truncate(true): +inode A is cut to 0 bytes, then rewritten + + +what rustc's mapping of inode A sees +rustc's mapping: 428 MB, MAP_PRIVATE, until it exits + + + + +inode A before: all 428 MB present + +new bytes + +gone: a touch here is SIGBUS +inode A is cut to 0 bytes, then refilled by the copy; +the mapping's private pages (relocations) go too: +SIGSEGV even when the new bytes are identical + + +the running rustc dies +SIGBUS when it touches a page past the new end +SIGSEGV through relocations the truncate threw +away, even when the new bytes are identical +A rustc starting during the 0.1 to 3 s copy +opens a half-written file. + +install_backend_into, all three writers +let tmp = temporary_backend_path(cache_dir); +// = librustc_codegen_cuda.so.<pid>.<nonce>.tmp +fs::copy(built_so, &tmp)?; // old .so and record stay +record_source_rev(cache_dir, None); // follow-up 1 +fs::rename(&tmp, &so_path)?; // atomic, same directory +write_toolchain_fingerprint(..); record_source_rev(..); +// any error: remove tmp, return it; the old .so stays + + +two inodes, one name +rustc's mapping stays on inode A until it exits + + + + +inode A: unnamed now, alive until its last mapper exits + +inode B: named librustc_codegen_cuda.so, already complete +a rustc starting now opens inode B, complete + +Two follow-ups +1 the record is cleared just before the rename, so + a failed copy keeps the old backend and its record +2 cargo oxide update sweeps a .tmp left by an install + killed between copy and rename; installs never do + +Why the rename is safe and the copy was not +rename(2) swaps the directory entry. A mapped inode stays allocated until its last mapper exits, and a new open sees the complete new file. +std::fs::copy reopened that inode with O_TRUNC under every mapping: pages past the new end are gone (SIGBUS), and so are the mapping's private pages (SIGSEGV). +Measured: a rustc with the 428 MB backend mapped, killed 5/5 in place (identical bytes), finished 5/5 by rename; an mmap reader of a 64 MB install, SIGBUS 6/6 before, 0/6 after. + diff --git a/cuda-oxide/cuda-oxide-book/getting-started/installation.md b/cuda-oxide/cuda-oxide-book/getting-started/installation.md index 5a6740b18d..2923e85c30 100644 --- a/cuda-oxide/cuda-oxide-book/getting-started/installation.md +++ b/cuda-oxide/cuda-oxide-book/getting-started/installation.md @@ -290,6 +290,15 @@ cargo +nightly-2026-10-11 install --git https://github.com/NVIDIA/cuda-rust.git On first run, `cargo-oxide` will automatically fetch and build the codegen backend. Subsequent runs reuse the cached build. +```{figure} images/backend-install-by-rename.svg +:align: center +:width: 100% + +Installing the cached backend by rename. Every `rustc` that compiles a kernel maps the one shared `.so`. +The installer copies a new build to a sibling temporary and renames it over the name, so running +compilers keep the old inode and a compiler that starts during the install opens a complete file. +``` + To discover the examples available in a cuda-oxide checkout: ```bash