Upgrade modern backend to LLVM 21.1.8 and CUDA 13.3

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,7 +22,7 @@ A clear and concise description of what you expected to happen.
 
 - OS: [e.g. Windows 11, Ubuntu 22.04]
 - GPU: [e.g. RTX 3060]
-- CUDA Toolkit version: [e.g. 13.2]
+- CUDA Toolkit version: [e.g. 13.3]
 - cuDNN version (if applicable): [e.g. 9.x]
 - Rust toolchain: [output of `rustc --version`]
 

.github/workflows/ci_linux.yml

@@ -28,16 +28,16 @@ jobs:
           - name: Ubuntu-24.04 / CUDA-12.8.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-24.04-arm
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / x86_64
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-latest
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / ARM64
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-24.04-arm
           - name: RockyLinux-9 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda12:latest"
             runner: ubuntu-latest
-          - name: RockyLinux-9 / CUDA-13.0.2 / x86_64
+          - name: RockyLinux-9 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda13:latest"
             runner: ubuntu-latest
 
@@ -152,8 +152,8 @@ jobs:
               --exclude cust
           '
 
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.

.github/workflows/ci_windows.yml

@@ -42,7 +42,7 @@ jobs:
               ]
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            cuda: "13.0.2"
+            cuda: "13.3.1"
             nvvm-dll-dir: "nvvm\\bin\\x64"
             sub-packages:
               [
@@ -65,7 +65,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install CUDA
-        uses: Jimver/cuda-toolkit@v0.2.29
+        uses: Jimver/cuda-toolkit@v0.2.36
         id: cuda-toolkit
         with:
           cuda: ${{ matrix.cuda }}
@@ -123,8 +123,8 @@ jobs:
             --exclude blastoff --exclude cudnn --exclude cudnn-sys --exclude cust
 
       # Exclude crates that require cuDNN, not available on Windows CI: cudnn, cudnn-sys.
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.

.github/workflows/container_images.yml

@@ -33,16 +33,16 @@ jobs:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
             dockerfile: ./container/ubuntu24-cuda12/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
             dockerfile: ./container/ubuntu24-cuda13/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
-            dockerfile: ./container/ubuntu24-cuda13-llvm19/Dockerfile
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
+            dockerfile: ./container/ubuntu24-cuda13-llvm21/Dockerfile
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
             dockerfile: ./container/rockylinux9-cuda12/Dockerfile
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
             dockerfile: ./container/rockylinux9-cuda13/Dockerfile
     steps:
@@ -161,13 +161,13 @@ jobs:
         variance:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
     steps:
       - name: Set lowercase repo owner

.github/workflows/llvm21.yml

@@ -0,0 +1,74 @@
+name: LLVM 21 experiment
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [experiment/cuda13.3-llvm21]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  vecadd:
+    name: CUDA 13.3 / LLVM 21 / ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: linux-x86_64
+          - runner: ubuntu-24.04-arm
+            arch: linux-aarch64
+    steps:
+      - uses: actions/checkout@v4
+
+      # Temporary experiment input until the LLVM 21 prebuilts have a release.
+      # Built from update/llvm-21.1.8 at 2cf27700bffbab35dff21a58d1e19ca12525ca6f.
+      - name: Download LLVM 21.1.8 prebuilt
+        env:
+          GH_TOKEN: ${{ github.token }}
+          LLVM_ARCH: ${{ matrix.arch }}
+        run: |
+          gh run download 34781677658 \
+            --repo brandonros/rustc_codegen_nvvm-llvm \
+            --name "$LLVM_ARCH-llvm-21.1.8" --dir llvm-prebuilt
+          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
+
+      - name: Build vector-add host and PTX with CUDA 13.3.1
+        env:
+          LLVM_ARCH: ${{ matrix.arch }}
+        run: |
+          docker run --rm \
+            -v "$PWD:/workspace" -w /workspace \
+            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
+            -e LLVM_LINK_STATIC=1 \
+            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
+            -e RUST_CUDA_EMIT_LLVM_IR=1 \
+            nvidia/cuda:13.3.1-devel-ubuntu24.04 bash -c '
+              set -euo pipefail
+              apt-get update
+              DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+                build-essential libclang-dev curl ca-certificates git pkg-config libssl-dev \
+                libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
+              curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
+                sh -s -- -y --profile minimal --default-toolchain none
+              export PATH="/root/.cargo/bin:$PATH"
+              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
+              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
+              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
+              nvcc --version
+              rustup show
+              cargo build -p vecadd --features llvm21
+            '
+
+      - name: Upload PTX and LLVM IR
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vecadd-llvm21-${{ matrix.arch }}
+          path: |
+            target/**/kernels.ptx
+            target/**/final-module.ll
+          if-no-files-found: warn

CONTRIBUTING.md

@@ -33,7 +33,7 @@ For questions, clarifications, and general help:
 ### Windows-Specific Notes
 
 - Ensure the CUDA Toolkit `bin` directory is on your `PATH` (e.g.
-  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin`).
+  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin`).
 - The MSVC build tools are required. Install via
   [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) with the
   "Desktop development with C++" workload.

container/rockylinux9-cuda13/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9 AS llvm-builder
 
 RUN dnf -y install \
     --nobest \
@@ -51,7 +51,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     rm -rf llvm-7.1.0.src* && \
     dnf clean all
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9
 
 RUN dnf -y install \
     --nobest \

container/ubuntu24-cuda13-llvm19/Dockerfile → container/ubuntu24-cuda13-llvm21/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -18,13 +18,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
 
-WORKDIR /data/llvm19
+WORKDIR /data/llvm21
 
-# Download and build LLVM 19.1.7 (the active LLVM 19 pin used by `rustc_codegen_nvvm`).
+# Download and build LLVM 21.1.8 (the active LLVM 21 pin used by `rustc_codegen_nvvm`).
 # LLVM 8+ ships as a monorepo tarball; cmake source root is the `llvm/` subdir.
-RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/llvm-project-19.1.7.src.tar.xz && \
-    tar -xf llvm-project-19.1.7.src.tar.xz && \
-    cd llvm-project-19.1.7.src && \
+RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.8/llvm-project-21.1.8.src.tar.xz && \
+    tar -xf llvm-project-21.1.8.src.tar.xz && \
+    cd llvm-project-21.1.8.src && \
     mkdir build && cd build && \
     ARCH=$(dpkg --print-architecture) && \
     if [ "$ARCH" = "amd64" ]; then \
@@ -44,14 +44,14 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
         -DLLVM_INCLUDE_BENCHMARKS=OFF \
         -DLLVM_ENABLE_ZLIB=ON \
         -DLLVM_ENABLE_TERMINFO=ON \
-        -DCMAKE_INSTALL_PREFIX=/opt/llvm-19 \
+        -DCMAKE_INSTALL_PREFIX=/opt/llvm-21 \
         ../llvm && \
     ninja -j$(nproc) && \
     ninja install && \
     cd ../.. && \
-    rm -rf llvm-project-19.1.7.src*
+    rm -rf llvm-project-21.1.8.src*
 
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -71,10 +71,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     libxrandr-dev && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=llvm-builder /opt/llvm-19 /opt/llvm-19
-RUN ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config && \
-    ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config-19 && \
-    ln -s /opt/llvm-19/bin/llvm-as /usr/bin/llvm-as-19
+COPY --from=llvm-builder /opt/llvm-21 /opt/llvm-21
+RUN ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config && \
+    ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config-21 && \
+    ln -s /opt/llvm-21/bin/llvm-as /usr/bin/llvm-as-21
 
 # Get Rust (install rustup; toolchain installed from rust-toolchain.toml below)
 RUN curl -sSf -L https://sh.rustup.rs | bash -s -- -y --profile minimal --default-toolchain none
@@ -85,15 +85,15 @@ WORKDIR /data/rust-cuda
 RUN --mount=type=bind,source=rust-toolchain.toml,target=/data/rust-cuda/rust-toolchain.toml \
     rustup show
 
-# Add nvvm + LLVM 19 dylib to the runtime linker path.
-ENV LD_LIBRARY_PATH="/opt/llvm-19/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
+# Add nvvm + LLVM 21 dylib to the runtime linker path.
+ENV LD_LIBRARY_PATH="/opt/llvm-21/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
 
-# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_19` to locate the LLVM 19
-# toolchain when the `llvm19` cargo feature is on. The feature itself is gated;
+# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_21` to locate the LLVM 21
+# toolchain when the `llvm21` cargo feature is on. The feature itself is gated;
 # downstream crates that depend on `cuda_builder` must build with
-# `--features llvm19` for this to take effect — that propagates through to
-# `nvvm/llvm19` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm19`
-# (LLVM 19 codegen path) per crates/cuda_builder/Cargo.toml.
-ENV LLVM_CONFIG_19=/opt/llvm-19/bin/llvm-config
+# `--features llvm21` for this to take effect — that propagates through to
+# `nvvm/llvm21` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm21`
+# (LLVM 21 codegen path) per crates/cuda_builder/Cargo.toml.
+ENV LLVM_CONFIG_21=/opt/llvm-21/bin/llvm-config
 ENV LLVM_LINK_STATIC=1
 ENV RUST_LOG=info

container/ubuntu24-cuda13/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -50,7 +50,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     cd ../.. && \
     rm -rf llvm-7.1.0.src*
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \

crates/cuda_builder/Cargo.toml

@@ -16,12 +16,12 @@ default = []
 # HACK(see rust-gpu/spirv-builder): use `dep:` to avoid Cargo auto-creating a feature
 # with the dependency name. Consumers must explicitly opt-in to compiling the backend.
 rustc_codegen_nvvm = ["dep:rustc_codegen_nvvm"]
-# Build the backend against LLVM 19 instead of LLVM 7. Propagates to `nvvm` (which
+# Build the backend against LLVM 21 instead of LLVM 7. Propagates to `nvvm` (which
 # uses it to flip the default `NvvmArch` to `Compute100`) and, when the optional
 # `rustc_codegen_nvvm` dep is also enabled, to `rustc_codegen_nvvm` itself. Even
 # when the optional dep is disabled, the build script's nested `cargo build -p
-# rustc_codegen_nvvm` reads `cfg(feature = "llvm19")` here and forwards it.
-llvm19 = ["nvvm/llvm19", "rustc_codegen_nvvm?/llvm19"]
+# rustc_codegen_nvvm` reads `cfg(feature = "llvm21")` here and forwards it.
+llvm21 = ["nvvm/llvm21", "rustc_codegen_nvvm?/llvm21"]
 
 [dependencies]
 rustc_codegen_nvvm = { version = "0.3", path = "../rustc_codegen_nvvm", optional = true }

crates/cuda_builder/src/lib.rs

@@ -556,11 +556,11 @@ fn build_backend_and_find(filename: &str) -> Option<PathBuf> {
         .arg(&target_dir)
         .current_dir(&workspace_dir);
 
-    // Propagate the `llvm19` cargo feature to the nested backend build. Without this
+    // Propagate the `llvm21` cargo feature to the nested backend build. Without this
     // `rustc_codegen_nvvm`'s build script falls through to the prebuilt LLVM 7
-    // download, which the LLVM 19 codegen path can't link against.
-    if cfg!(feature = "llvm19") {
-        cmd.args(["--features", "llvm19"]);
+    // download, which the LLVM 21 codegen path can't link against.
+    if cfg!(feature = "llvm21") {
+        cmd.args(["--features", "llvm21"]);
     }
 
     let status = cmd.status().ok()?;

crates/cuda_std/src/warp.rs

@@ -744,7 +744,7 @@ pub enum WarpShuffleMode {
 // The libintrinsics.ll wrappers pack their (value, predicate) result into a
 // single i64: low 32 bits = value, bit 32 = predicate. Returning a primitive
 // integer avoids the small-aggregate ABI path where rustc attaches `align N`
-// to the call's return value — an attribute LLVM 19's verifier rejects on
+// to the call's return value — an attribute LLVM 21's verifier rejects on
 // non-pointer returns.
 // Unused on host targets — every caller is `#[gpu_only]`.
 #[allow(dead_code)]

crates/nvvm/Cargo.toml

@@ -10,11 +10,11 @@ readme = "../../README.md"
 
 [features]
 default = []
-# Match the `llvm19` feature on `rustc_codegen_nvvm`. Currently only flips the
+# Match the `llvm21` feature on `rustc_codegen_nvvm`. Currently only flips the
 # default `NvvmArch` to the lowest Blackwell capability, since the LLVM 7
-# bitcode dialect can't target `compute_100+` and the LLVM 19 dialect can't
+# bitcode dialect can't target `compute_100+` and the LLVM 21 dialect can't
 # target pre-Blackwell archs.
-llvm19 = []
+llvm21 = []
 
 [dependencies]
 cust_raw = { version = "0.11.3", path = "../cust_raw", default-features = false, features = ["nvvm"] }