Add verified LLVM cleanup passes with retention and replay regressions

.github/workflows/ptx_export.yml

@@ -24,6 +24,14 @@ jobs:
           git rev-parse HEAD > artifacts/ptx/source-commit.txt
           nix develop .#v21 --command rustc -Vv > artifacts/ptx/rustc-version.txt
           sha256sum artifacts/ptx/rust_kernels.ptx > artifacts/ptx/SHA256SUMS
+      - name: Check Python regressions
+        run: python3 -B -m unittest discover -s examples/ptx_export -p 'test_*.py'
+      - name: Check guarded-select semantics
+        run: |
+          nix develop .#v21 --command rustc --edition=2024 --test examples/ptx_export/kernels/src/guarded_select.rs -o /tmp/guarded-select
+          /tmp/guarded-select
+      - name: Check default DCE retention
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_default_dce.py artifacts/ptx
       - uses: actions/upload-artifact@v4
         with:
           name: rust-ptx

Cargo.toml

@@ -53,6 +53,7 @@ members = [
   "examples/vecadd/kernels",
   "examples/ptx_export",
   "examples/ptx_export/kernels",
+  "examples/ptx_export/retention-kernels",
 
   "samples/introduction/async_api",
   "samples/introduction/async_api/kernels",

crates/cuda_builder/src/lib.rs

@@ -51,6 +51,18 @@ impl DebugInfo {
     }
 }
 
+/// Experimental pre-NVVM optimization. Requires the LLVM 21 backend.
+/// The historical LLVM 19 API names are retained for caller compatibility.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Llvm19Cleanup {
+    /// Remove unreachable internal definitions without rewriting live function bodies.
+    GlobalDce,
+    /// Inline internal calls, then run scalar cleanup without correlated propagation.
+    InlineScalar,
+    Scalar,
+    Inline,
+}
+
 pub enum EmitOption {
     LlvmIr,
     Bitcode,
@@ -194,6 +206,12 @@ pub struct CudaBuilder {
     /// An optional path where to dump LLVM IR of the final output the codegen will feed to libnvvm. Usually
     /// used for debugging.
     pub final_module_path: Option<PathBuf>,
+    /// Whether the modern backend removes unreachable definitions at the merged handoff.
+    pub llvm19_global_dce: bool,
+    /// Additional opt-in modern LLVM cleanup; disabled by default.
+    pub llvm19_cleanup: Option<Llvm19Cleanup>,
+    /// Experimental scalar cleanup of each codegen unit before serialization.
+    pub llvm19_module_cleanup: bool,
 }
 
 impl CudaBuilder {
@@ -216,9 +234,33 @@ impl CudaBuilder {
             debug: DebugInfo::None,
             build_args: vec![],
             final_module_path: None,
+            llvm19_global_dce: true,
+            llvm19_cleanup: None,
+            llvm19_module_cleanup: false,
         }
     }
 
+    /// Enable or disable the default modern LLVM merged-module GlobalDCE pass.
+    /// Disabling is intended for compiler-output comparisons; LLVM 7 is unchanged.
+    pub fn llvm19_global_dce(mut self, enabled: bool) -> Self {
+        self.llvm19_global_dce = enabled;
+        self
+    }
+
+    /// Enable verified scalar cleanup before each codegen unit is serialized.
+    /// Disabled by default; independent of merged-module cleanup.
+    pub fn llvm19_module_cleanup(mut self, enabled: bool) -> Self {
+        self.llvm19_module_cleanup = enabled;
+        self
+    }
+
+    /// Enable a bounded modern LLVM cleanup pipeline before NVVM compilation.
+    /// This is experimental; compare numerical results and generated code.
+    pub fn llvm19_cleanup(mut self, cleanup: Llvm19Cleanup) -> Self {
+        self.llvm19_cleanup = Some(cleanup);
+        self
+    }
+
     /// Additional arguments passed to cargo during `cargo build`.
     pub fn build_args(mut self, args: &[impl AsRef<str>]) -> Self {
         self.build_args
@@ -723,6 +765,21 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result<PathBuf, CudaBuilderError> {
     }
 
     let mut llvm_args = vec![NvvmOption::Arch(builder.arch).to_string()];
+    if !builder.llvm19_global_dce {
+        llvm_args.push("--disable-llvm19-global-dce".to_string());
+    }
+    if builder.llvm19_module_cleanup {
+        llvm_args.push("--llvm19-module-cleanup".to_string());
+    }
+    if let Some(mode) = builder.llvm19_cleanup {
+        let mode = match mode {
+            Llvm19Cleanup::GlobalDce => "dce",
+            Llvm19Cleanup::InlineScalar => "inline-scalar",
+            Llvm19Cleanup::Scalar => "scalar",
+            Llvm19Cleanup::Inline => "inline",
+        };
+        llvm_args.push(format!("--llvm19-cleanup={mode}"));
+    }
 
     if !builder.nvvm_opts {
         llvm_args.push("-opt=0".to_string());

crates/rustc_codegen_nvvm/build.rs

@@ -306,7 +306,11 @@ fn rustc_llvm_build(flavor: &LlvmFlavor) {
 
     configure_libintrinsics(&llvm_config, flavor);
 
-    let required_components = &["ipo", "bitreader", "bitwriter", "lto", "nvptx"];
+    let required_components: &[&str] = if flavor.major >= 19 {
+        &["ipo", "bitreader", "bitwriter", "lto", "nvptx", "passes"]
+    } else {
+        &["ipo", "bitreader", "bitwriter", "lto", "nvptx"]
+    };
 
     let components = output(Command::new(&llvm_config).arg("--components"));
     let mut components = components.split_whitespace().collect::<Vec<_>>();

crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp

@@ -13,6 +13,7 @@
 #include <vector>
 #include <set>
 #include <optional>
+#include <memory>
 #include <string>
 
 #include "rustllvm.h"
@@ -25,6 +26,9 @@
 #include "llvm/Support/FileSystem.h"
 #if LLVM_VERSION_MAJOR >= 19
 #include "llvm/Transforms/IPO/Internalize.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Support/Error.h"
 #endif
 #if LLVM_VERSION_MAJOR >= 19
 #include "llvm/TargetParser/Host.h"
@@ -166,6 +170,75 @@ extern "C" void LLVMPassManagerBuilderPopulateLTOPassManager(
 }
 #endif
 
+// Explicit, bounded modern-PM cleanup at the final NVVM handoff. This is
+// separate from the legacy compatibility builder and remains opt-in.
+// Keep discriminants in sync with llvm::NvvmCleanup on the Rust side.
+enum class LLVMRustNvvmCleanup : uint32_t { Scalar = 0, Inline = 1, GlobalDce = 2, InlineScalar = 3 };
+
+extern "C" LLVMRustResult LLVMRustRunNvvmCleanup(LLVMModuleRef M, LLVMRustNvvmCleanup Mode)
+{
+#if LLVM_VERSION_MAJOR >= 19
+  Module &Mod = *unwrap(M);
+  std::string Diagnostics;
+  raw_string_ostream OS(Diagnostics);
+  if (verifyModule(Mod, &OS)) {
+    LLVMRustSetLastError(OS.str().c_str());
+    return LLVMRustResult::Failure;
+  }
+  LoopAnalysisManager LAM;
+  FunctionAnalysisManager FAM;
+  CGSCCAnalysisManager CGAM;
+  ModuleAnalysisManager MAM;
+  // Match opt's target-aware analyses. Without a TargetMachine the inliner
+  // uses generic costs and can disagree with replay even on identical IR.
+  // Use the module's NVPTX triple and generic CPU, as the existing backend
+  // does; NVVM remains responsible for the selected compute architecture.
+  std::string TargetError;
+  const Target *T = TargetRegistry::lookupTarget(Mod.getTargetTriple(), TargetError);
+  if (!T) {
+    LLVMRustSetLastError(TargetError.c_str());
+    return LLVMRustResult::Failure;
+  }
+  std::unique_ptr<TargetMachine> TM(T->createTargetMachine(
+      Mod.getTargetTriple(), "", "", TargetOptions(), std::nullopt));
+  if (!TM) {
+    LLVMRustSetLastError("Could not create cleanup TargetMachine");
+    return LLVMRustResult::Failure;
+  }
+  PassBuilder PB(TM.get());
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+  ModulePassManager PM;
+  // Prune unreachable functions before inlining, then expose and simplify
+  // branch-correlated iterator values. Keep this identical to offline replay.
+  const char *Pipeline = Mode == LLVMRustNvvmCleanup::GlobalDce
+      ? "globaldce,verify"
+      : Mode == LLVMRustNvvmCleanup::InlineScalar
+      ? "globaldce,cgscc(inline),function(sroa,instcombine<max-iterations=2;no-verify-fixpoint>,simplifycfg,adce),globaldce,verify"
+      : Mode == LLVMRustNvvmCleanup::Inline
+      ? "globaldce,cgscc(inline),function(sroa,instcombine<max-iterations=2;no-verify-fixpoint>,simplifycfg,adce),"
+        "globaldce,function(correlated-propagation,instcombine<max-iterations=2;no-verify-fixpoint>,simplifycfg,adce),verify"
+      : "function(sroa,instcombine<max-iterations=2;no-verify-fixpoint>,simplifycfg,adce),verify";
+  if (auto Error = PB.parsePassPipeline(PM, Pipeline)) {
+    LLVMRustSetLastError(toString(std::move(Error)).c_str());
+    return LLVMRustResult::Failure;
+  }
+  PM.run(Mod, MAM);
+  Diagnostics.clear();
+  if (verifyModule(Mod, &OS)) {
+    LLVMRustSetLastError(OS.str().c_str());
+    return LLVMRustResult::Failure;
+  }
+  return LLVMRustResult::Success;
+#else
+  LLVMRustSetLastError("NVVM cleanup requires LLVM 19");
+  return LLVMRustResult::Failure;
+#endif
+}
+
 extern "C" void LLVMInitializePasses()
 {
 #if LLVM_VERSION_MAJOR >= 19

crates/rustc_codegen_nvvm/src/back.rs

@@ -328,6 +328,47 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen
             }
         }
 
+        // Run only after definitions, used globals and debug metadata are
+        // finalized. Scalar cleanup preserves cross-module linkage; no DCE or
+        // inlining is performed here. Merged-module cleanup remains independent.
+        let args = crate::context::CodegenArgs::from_session(tcx.sess);
+        if args.llvm19_module_cleanup {
+            let llmod = unsafe { &*llvm_module.llmod };
+            let dump = |stage: &str| {
+                if let Some(final_path) = &args.final_module_path {
+                    let directory = final_path.parent().unwrap().join("per-module");
+                    std::fs::create_dir_all(&directory).unwrap_or_else(|error| {
+                        tcx.sess
+                            .dcx()
+                            .fatal(format!("cannot create module IR directory: {error}"))
+                    });
+                    let path = directory.join(format!("{cgu_name}.{stage}.ll"));
+                    let path = path.to_str().unwrap();
+                    unsafe {
+                        llvm::LLVMRustPrintModule(
+                            llmod,
+                            path.as_c_char_ptr(),
+                            path.len(),
+                            demangle_callback,
+                        )
+                        .into_result()
+                        .unwrap_or_else(|_| {
+                            llvm_err(tcx.sess.dcx(), "cannot save per-module IR").raise();
+                        });
+                    }
+                }
+            };
+            dump("before");
+            unsafe {
+                llvm::LLVMRustRunNvvmCleanup(llmod, llvm::NvvmCleanup::Scalar)
+                    .into_result()
+                    .unwrap_or_else(|_| {
+                        llvm_err(tcx.sess.dcx(), "LLVM 19 per-module cleanup failed").raise();
+                    });
+            }
+            dump("after");
+        }
+
         ModuleCodegen::new_regular(cgu_name.to_string(), llvm_module)
     }
 

crates/rustc_codegen_nvvm/src/context.rs

@@ -651,6 +651,10 @@ pub struct CodegenArgs {
     pub override_libm: bool,
     pub use_constant_memory_space: bool,
     pub final_module_path: Option<PathBuf>,
+    // None leaves the existing NVVM handoff unchanged.
+    pub llvm19_cleanup: Option<crate::llvm::NvvmCleanup>,
+    pub llvm19_module_cleanup: bool,
+    pub disable_llvm19_global_dce: bool,
     pub disassemble: Option<DisassembleMode>,
 }
 
@@ -675,6 +679,32 @@ impl CodegenArgs {
                 cg_args.override_libm = true;
             } else if arg == "--use-constant-memory-space" {
                 cg_args.use_constant_memory_space = true;
+            } else if arg == "--disable-llvm19-global-dce" {
+                if !cfg!(feature = "llvm21") {
+                    sess.dcx()
+                        .fatal("--disable-llvm19-global-dce requires the llvm21 backend feature");
+                }
+                cg_args.disable_llvm19_global_dce = true;
+            } else if arg == "--llvm19-module-cleanup" {
+                if !cfg!(feature = "llvm21") {
+                    sess.dcx()
+                        .fatal("--llvm19-module-cleanup requires the llvm21 backend feature");
+                }
+                cg_args.llvm19_module_cleanup = true;
+            } else if let Some(mode) = arg.strip_prefix("--llvm19-cleanup=") {
+                if !cfg!(feature = "llvm21") {
+                    sess.dcx()
+                        .fatal("--llvm19-cleanup requires the llvm21 backend feature");
+                }
+                cg_args.llvm19_cleanup = Some(match mode {
+                    "scalar" => crate::llvm::NvvmCleanup::Scalar,
+                    "inline" => crate::llvm::NvvmCleanup::Inline,
+                    "dce" => crate::llvm::NvvmCleanup::GlobalDce,
+                    "inline-scalar" => crate::llvm::NvvmCleanup::InlineScalar,
+                    _ => sess
+                        .dcx()
+                        .fatal("--llvm19-cleanup expects scalar, inline, inline-scalar, or dce"),
+                });
             } else if arg == "--final-module-path" {
                 let path = match args.get(idx + 1) {
                     Some(p) => p,

crates/rustc_codegen_nvvm/src/llvm.rs

@@ -28,6 +28,16 @@ use std::ptr::{self};
 use crate::{builder::unnamed, common::AsCCharPtr};
 pub use debuginfo::*;
 
+// Keep discriminants in sync with LLVMRustNvvmCleanup in PassWrapper.cpp.
+#[repr(u32)]
+#[derive(Clone, Copy)]
+pub enum NvvmCleanup {
+    Scalar = 0,
+    Inline = 1,
+    GlobalDce = 2,
+    InlineScalar = 3,
+}
+
 impl PartialEq for Value {
     fn eq(&self, other: &Self) -> bool {
         ptr::eq(self, other)
@@ -1331,6 +1341,7 @@ unsafe extern "C" {
     ) -> Option<&'a DILocation>;
 
     pub(crate) fn LLVMRustRunFunctionPassManager(PM: &PassManager, M: &Module);
+    pub(crate) fn LLVMRustRunNvvmCleanup(M: &Module, mode: NvvmCleanup) -> LLVMRustResult;
     pub(crate) fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool);
 
     pub(crate) fn LLVMRustAddBuilderLibraryInfo(