Archive LLVM 19 optimization measurements and research reports

.github/workflows/optimization_wave2.yml

@@ -0,0 +1,68 @@
+name: LLVM 21 optimization wave 2
+
+on:
+  workflow_dispatch:
+    inputs:
+      workload:
+        description: Workload to repeat (all runs the complete matrix)
+        type: choice
+        default: all
+        options: [all, small, representative, solana, bitcoin, ethereum, shallenge, self_test]
+  push:
+    branches: [poc/portable-ptx-export]
+    paths:
+      - '.github/workflows/optimization_wave2.yml'
+      - 'examples/ptx_export/run_wave2.py'
+      - 'examples/ptx_export/representative-kernels/**'
+
+permissions:
+  contents: read
+
+jobs:
+  compare:
+    name: Compare ${{ matrix.workload }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: ${{ matrix.workload == 'self_test' && 90 || 45 }}
+    strategy:
+      fail-fast: false
+      matrix:
+        workload: ${{ fromJSON(inputs.workload != '' && inputs.workload != 'all' && format('["{0}"]', inputs.workload) || '["small","representative","solana","bitcoin","ethereum","shallenge","self_test"]') }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@v14
+        with:
+          use-flakehub: false
+      - uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: wave2-cargo-${{ runner.os }}-llvm21-${{ github.sha }}
+          restore-keys: |
+            ptx-cargo-v1-${{ runner.os }}-${{ runner.arch }}-llvm21-
+      - name: Build the checked-out backend
+        run: |
+          nix develop .#v21 --command cargo build -p rustc_codegen_nvvm --features llvm21 --target-dir target/cuda-builder-codegen
+          rm -rf target/nvptx64-nvidia-cuda
+      - name: Checkout pinned workload
+        uses: actions/checkout@v4
+        with:
+          repository: brandonros/vanity-miner-rs
+          ref: 9791234249fc8cb762c296c4fda4503d2686ff77
+          path: workloads/vanity-miner
+      - name: Compile and compare
+        run: |
+          mkdir -p artifacts
+          git rev-parse HEAD > artifacts/source-commit.txt
+          sha256sum target/cuda-builder-codegen/debug/librustc_codegen_nvvm.so > artifacts/backend-sha256.txt
+          nix develop .#v21 --command rustc -Vv > artifacts/rustc-version.txt
+          cp flake.lock Cargo.lock rust-toolchain.toml artifacts/
+          nix develop .#v21 --command python3 examples/ptx_export/run_wave2.py ${{ matrix.workload }} --out artifacts/${{ matrix.workload }} --miner workloads/vanity-miner
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: wave2-${{ matrix.workload }}
+          path: artifacts/
+          if-no-files-found: error

.github/workflows/ptx_export.yml

@@ -0,0 +1,123 @@
+name: Export portable PTX
+
+on:
+  pull_request:
+    branches: [experiment/cuda13.3-llvm21, "stack/**"]
+  workflow_dispatch:
+    inputs:
+      cleanup_experiment:
+        description: Replay LLVM 21 handoff IR with opt-in cleanup passes
+        type: boolean
+        default: false
+      optimization_sweep:
+        description: Run the extended CFG, DCE, memory and inlining experiments
+        type: boolean
+        default: false
+      mining_workload:
+        description: Compile pinned Solana mining source and replay memory cleanup
+        type: boolean
+        default: false
+  push:
+    branches: [poc/portable-ptx-export]
+
+permissions:
+  contents: read
+
+jobs:
+  export:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@v14
+        with:
+          use-flakehub: false
+      - name: Restore Cargo cache
+        id: cargo-cache
+        uses: actions/cache/restore@v4
+        with:
+          # Include all nested builds, notably target/cuda-builder-codegen.
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: ptx-cargo-v1-${{ runner.os }}-${{ runner.arch }}-llvm21-${{ hashFiles('flake.nix', 'flake.lock', 'rust-toolchain.toml', '**/Cargo.toml', '**/Cargo.lock', '.cargo/**', '.github/workflows/ptx_export.yml') }}-${{ github.sha }}
+          restore-keys: |
+            ptx-cargo-v1-${{ runner.os }}-${{ runner.arch }}-llvm21-${{ hashFiles('flake.nix', 'flake.lock', 'rust-toolchain.toml', '**/Cargo.toml', '**/Cargo.lock', '.cargo/**', '.github/workflows/ptx_export.yml') }}-
+      - name: Test codegen inventory parser
+        run: python3 -m unittest discover -s examples/ptx_export -p 'test_*.py'
+      - name: Rebuild the checked-out backend and regenerate device evidence
+        run: |
+          # cuda_builder searches for an existing backend before trying Cargo.
+          # A restored .so alone therefore does not establish source freshness.
+          nix develop .#v21 --command cargo build -p rustc_codegen_nvvm --features llvm21 --target-dir target/cuda-builder-codegen
+          # Device Cargo fingerprints do not track changes to the backend .so;
+          # cached builds also omit the IR side outputs required by this job.
+          # Retain the expensive host/backend dependency cache.
+          rm -rf target/nvptx64-nvidia-cuda
+      - name: Check guarded-select source semantics on CPU
+        run: |
+          nix develop .#v21 --command rustc --edition=2024 --test examples/ptx_export/kernels/src/guarded_select.rs -o /tmp/guarded-select-debug
+          /tmp/guarded-select-debug
+          nix develop .#v21 --command rustc --edition=2024 -O --test examples/ptx_export/kernels/src/guarded_select.rs -o /tmp/guarded-select-release
+          /tmp/guarded-select-release
+      - name: Compile Rust kernels without a GPU
+        id: export
+        run: |
+          mkdir -p artifacts/ptx
+          printf '%s\n' 'nix develop .#v21 --command cargo run -vv -p ptx_export --features llvm21 -- artifacts/ptx none' > artifacts/ptx/build-command.txt
+          nix develop .#v21 --command cargo run -vv -p ptx_export --features llvm21 -- artifacts/ptx none 2>&1 | tee artifacts/ptx/build.log
+      - name: Verify LLVM 21 handoff IR
+        run: nix develop .#v21 --command opt-21 -passes=verify -disable-output artifacts/ptx/final-module.ll
+      - name: Record provenance
+        run: |
+          git rev-parse HEAD > artifacts/ptx/source-commit.txt
+          nix develop .#v21 --command rustc -Vv > artifacts/ptx/rustc-version.txt
+          nix develop .#v21 --command nvcc --version > artifacts/ptx/nvcc-version.txt
+          cp flake.lock rust-toolchain.toml Cargo.lock artifacts/ptx/
+          sha256sum target/cuda-builder-codegen/debug/librustc_codegen_nvvm.so > artifacts/ptx/backend-sha256.txt
+          # Keep per-crate LLVM IR as well as the linked final-module.ll.
+          mkdir -p artifacts/ptx/llvm-ir
+          find target/nvptx64-nvidia-cuda -name '*.ll' -exec cp --parents '{}' artifacts/ptx/llvm-ir/ \;
+      - name: Assemble and inspect NVIDIA machine code without a GPU
+        run: nix develop .#v21 --command python3 examples/ptx_export/inspect_codegen.py artifacts/ptx
+      - name: Verify default DCE retention and explicit disable
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_default_dce.py artifacts/ptx
+      - name: Investigate pre-NVVM cleanup (opt-in)
+        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.cleanup_experiment || inputs.optimization_sweep) }}
+        run: nix develop .#v21 --command python3 examples/ptx_export/replay_cleanup.py artifacts/ptx ${{ inputs.optimization_sweep && '--extended' || '' }}
+      - name: Validate integrated cleanup against replay (opt-in)
+        if: ${{ github.event_name == 'workflow_dispatch' && (inputs.cleanup_experiment || inputs.optimization_sweep) }}
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_integrated_cleanup.py artifacts/ptx
+      - name: Validate per-module cleanup (extended sweep)
+        if: ${{ !cancelled() && steps.export.outcome == 'success' && github.event_name == 'workflow_dispatch' && inputs.optimization_sweep }}
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_module_cleanup.py artifacts/ptx
+      - name: Validate size-oriented builds (extended sweep)
+        if: ${{ !cancelled() && steps.export.outcome == 'success' && github.event_name == 'workflow_dispatch' && inputs.optimization_sweep }}
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_workloads.py artifacts/ptx
+      - name: Checkout pinned mining workload
+        if: ${{ !cancelled() && steps.export.outcome == 'success' && github.event_name == 'workflow_dispatch' && inputs.mining_workload }}
+        uses: actions/checkout@v4
+        with:
+          repository: brandonros/vanity-miner-rs
+          ref: 9791234249fc8cb762c296c4fda4503d2686ff77
+          path: workloads/vanity-miner
+      - name: Compare cleanup on Solana mining kernel
+        if: ${{ !cancelled() && steps.export.outcome == 'success' && github.event_name == 'workflow_dispatch' && inputs.mining_workload }}
+        run: nix develop .#v21 --command python3 examples/ptx_export/check_workloads.py artifacts/ptx --miner workloads/vanity-miner
+      - uses: actions/upload-artifact@v4
+        if: ${{ !cancelled() }}
+        with:
+          name: rust-ptx
+          path: artifacts/ptx/
+          if-no-files-found: error
+      - name: Save Cargo cache, including after compilation failures
+        if: ${{ !cancelled() && steps.cargo-cache.outcome == 'success' && steps.cargo-cache.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@v4
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            target/
+          key: ${{ steps.cargo-cache.outputs.cache-primary-key }}

Cargo.toml

@@ -51,6 +51,10 @@ members = [
   "examples/sha2_crates_io/kernels",
   "examples/vecadd",
   "examples/vecadd/kernels",
+  "examples/ptx_export",
+  "examples/ptx_export/kernels",
+  "examples/ptx_export/retention-kernels",
+  "examples/ptx_export/representative-kernels",
 
   "samples/introduction/async_api",
   "samples/introduction/async_api/kernels",

crates/cuda_builder/src/lib.rs

@@ -51,6 +51,18 @@ impl DebugInfo {
     }
 }
 
+/// Experimental pre-NVVM optimization. Requires the LLVM 21 backend.
+/// The historical LLVM 19 API names are retained for caller compatibility.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Llvm19Cleanup {
+    /// Remove unreachable internal definitions without rewriting live function bodies.
+    GlobalDce,
+    /// Inline internal calls, then run scalar cleanup without correlated propagation.
+    InlineScalar,
+    Scalar,
+    Inline,
+}
+
 pub enum EmitOption {
     LlvmIr,
     Bitcode,
@@ -194,6 +206,12 @@ pub struct CudaBuilder {
     /// An optional path where to dump LLVM IR of the final output the codegen will feed to libnvvm. Usually
     /// used for debugging.
     pub final_module_path: Option<PathBuf>,
+    /// Whether the modern backend removes unreachable definitions at the merged handoff.
+    pub llvm19_global_dce: bool,
+    /// Additional opt-in modern LLVM cleanup; disabled by default.
+    pub llvm19_cleanup: Option<Llvm19Cleanup>,
+    /// Experimental scalar cleanup of each codegen unit before serialization.
+    pub llvm19_module_cleanup: bool,
 }
 
 impl CudaBuilder {
@@ -216,9 +234,33 @@ impl CudaBuilder {
             debug: DebugInfo::None,
             build_args: vec![],
             final_module_path: None,
+            llvm19_global_dce: true,
+            llvm19_cleanup: None,
+            llvm19_module_cleanup: false,
         }
     }
 
+    /// Enable or disable the default modern LLVM merged-module GlobalDCE pass.
+    /// Disabling is intended for compiler-output comparisons; LLVM 7 is unchanged.
+    pub fn llvm19_global_dce(mut self, enabled: bool) -> Self {
+        self.llvm19_global_dce = enabled;
+        self
+    }
+
+    /// Enable verified scalar cleanup before each codegen unit is serialized.
+    /// Disabled by default; independent of merged-module cleanup.
+    pub fn llvm19_module_cleanup(mut self, enabled: bool) -> Self {
+        self.llvm19_module_cleanup = enabled;
+        self
+    }
+
+    /// Enable a bounded modern LLVM cleanup pipeline before NVVM compilation.
+    /// This is experimental; compare numerical results and generated code.
+    pub fn llvm19_cleanup(mut self, cleanup: Llvm19Cleanup) -> Self {
+        self.llvm19_cleanup = Some(cleanup);
+        self
+    }
+
     /// Additional arguments passed to cargo during `cargo build`.
     pub fn build_args(mut self, args: &[impl AsRef<str>]) -> Self {
         self.build_args
@@ -723,6 +765,21 @@ fn invoke_rustc(builder: &CudaBuilder) -> Result<PathBuf, CudaBuilderError> {
     }
 
     let mut llvm_args = vec![NvvmOption::Arch(builder.arch).to_string()];
+    if !builder.llvm19_global_dce {
+        llvm_args.push("--disable-llvm19-global-dce".to_string());
+    }
+    if builder.llvm19_module_cleanup {
+        llvm_args.push("--llvm19-module-cleanup".to_string());
+    }
+    if let Some(mode) = builder.llvm19_cleanup {
+        let mode = match mode {
+            Llvm19Cleanup::GlobalDce => "dce",
+            Llvm19Cleanup::InlineScalar => "inline-scalar",
+            Llvm19Cleanup::Scalar => "scalar",
+            Llvm19Cleanup::Inline => "inline",
+        };
+        llvm_args.push(format!("--llvm19-cleanup={mode}"));
+    }
 
     if !builder.nvvm_opts {
         llvm_args.push("-opt=0".to_string());

crates/cuda_std/src/warp.rs

@@ -8,6 +8,9 @@ use crate::gpu_only;
 use core::arch::asm;
 use half::{bf16, f16};
 
+#[path = "warp_control.rs"]
+mod control;
+
 /// Synchronizes all of the threads inside of this warp according to `mask`.
 ///
 /// # Safety
@@ -766,15 +769,7 @@ unsafe fn warp_shuffle_32(
         fn __nvvm_warp_shuffle(mask: u32, mode: u32, a: u32, b: u32, c: u32) -> u64;
     }
 
-    assert!(
-        !(width & (width - 1)) != 0 && width <= 32,
-        "width must be a power of 2 and less than or equal to 32"
-    );
-
-    // mimicking nvcc's behavior
-    let mut c = 0;
-    c |= 0b11111;
-    c |= (32 - width) << 8;
+    let c = control::shuffle_control(width, matches!(mode, WarpShuffleMode::Up));
 
     let result = unsafe { __nvvm_warp_shuffle(mask, mode as u32, value, b, c) };
     unpack_warp_result(result)