Align CUDA packages and images with driver-only runtime support

.github/workflows/package-image-row.yml

@@ -460,6 +460,8 @@ jobs:
             MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }}
             MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }}
             MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }}
+            NVIDIA_VISIBLE_DEVICES=${{ fromJSON(inputs.row_json).nvidia_visible_devices }}
+            NVIDIA_DRIVER_CAPABILITIES=${{ fromJSON(inputs.row_json).nvidia_driver_capabilities }}
       - name: Record Depot dry-image measurement
         if: always()
         env:
@@ -680,6 +682,8 @@ jobs:
             MESH_LLM_HOST_SHA=${{ steps.product.outputs.host_sha }}
             MESH_LLM_RUNTIME_ID=${{ steps.product.outputs.runtime_id }}
             MESH_LLM_RUNTIME_SHA=${{ steps.product.outputs.runtime_sha }}
+            NVIDIA_VISIBLE_DEVICES=${{ fromJSON(inputs.row_json).nvidia_visible_devices }}
+            NVIDIA_DRIVER_CAPABILITIES=${{ fromJSON(inputs.row_json).nvidia_driver_capabilities }}
       - name: Record Depot staged-image measurement
         if: always()
         env:

TODO.md

@@ -1,5 +1,23 @@
 # Production Readiness TODO
 
+- [x] Align CUDA packages and images with driver-only native runtime support.
+  Final result: no CUDA row declares `cuda-cudart`, cuBLAS, NCCL, or Arch `cuda`
+  as a package dependency or installs them into the image, because upstream
+  packages that closure into the native runtime and verifies it with
+  `LD_LIBRARY_PATH` unset. Ubuntu CUDA rows build on `ubuntu:24.04` instead of
+  `nvidia/cuda`, and the matrix derives `NVIDIA_VISIBLE_DEVICES` and
+  `NVIDIA_DRIVER_CAPABILITIES` per row so the container toolkit can still inject
+  the host driver; Arch CUDA gains those variables for the first time. There is
+  deliberately no `NVIDIA_REQUIRE_CUDA` replacement, since the old value
+  described the base image's toolkit rather than the packaged runtime's driver
+  floor.
+  QA: package metadata fixtures prove Ubuntu and Arch CUDA control files name no
+  NVIDIA user-space package and that the assertion fails when one is
+  reintroduced; runtime dependency fixtures prove CUDA rows install exactly the
+  CPU set; matrix tests prove the plain base and the derived toolkit variables
+  at 100% coverage; `image-matrix.ts validate`, `buildx --check` for all four
+  backends, shellcheck, and actionlint pass.
+
 - [ ] Keep no-driver package QA reliable under hosted-runner load while still
   enforcing bounded SIGINT shutdown. QA: the shared Linux smoke and Homebrew
   formula use the same 30-second bound, focused regression tests pass, the full

docker/Dockerfile.mesh-llm

@@ -50,7 +50,7 @@ RUN --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BA
     --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-apk-packages,target=/var/cache/apk,sharing=locked \
     --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-pacman-packages,target=/var/cache/pacman/pkg,sharing=locked \
     --mount=type=cache,id=mesh-packaging-${DISTRO}-${TARGETARCH}-${BACKEND}-${BACKEND_VERSION}-pacman-index,target=/var/lib/pacman/sync,sharing=locked \
-    sh /usr/local/bin/install-runtime-deps.sh "${DISTRO}" "${BACKEND}" "${BACKEND_VERSION}"
+    sh /usr/local/bin/install-runtime-deps.sh "${DISTRO}" "${BACKEND}"
 
 FROM runtime-deps AS runtime
 ARG PACKAGE_FILE
@@ -62,6 +62,17 @@ ARG MESH_LLM_RUNTIME_ID
 ARG MESH_LLM_RUNTIME_SHA
 ARG MESH_LLM_REPOSITORY=Mesh-LLM/mesh-llm
 ARG SOURCE_DATE_EPOCH
+# CUDA rows used to inherit these from the nvidia/cuda base. They are what the
+# container toolkit reads to inject the host driver, so a plain Ubuntu base has
+# to declare them itself. The matrix supplies them only for NVIDIA rows; empty
+# is inert and reads the same as unset. There is deliberately no
+# NVIDIA_REQUIRE_CUDA replacement: that value described the base image's
+# toolkit, and the packaged runtime now carries its own toolkit closure, so the
+# driver is the only remaining host requirement.
+ARG NVIDIA_VISIBLE_DEVICES=
+ARG NVIDIA_DRIVER_CAPABILITIES=
+ENV NVIDIA_VISIBLE_DEVICES=${NVIDIA_VISIBLE_DEVICES} \
+    NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES}
 LABEL org.opencontainers.image.title="mesh-llm" \
       org.opencontainers.image.description="Packaged mesh-llm runtime" \
       org.opencontainers.image.source="https://github.com/Mesh-LLM/mesh-packaging" \

docker/install-runtime-deps.sh

@@ -3,7 +3,6 @@ set -eu
 
 distro="${1:?distro is required}"
 backend="${2:?backend is required}"
-backend_version="${3:-}"
 
 disable_pacman_sandbox() {
   if ! grep -q '^DisableSandbox$' /etc/pacman.conf; then
@@ -28,18 +27,8 @@ case "$distro" in
     if [ "$backend" = "vulkan" ]; then
       set -- "$@" libvulkan1
     fi
-    if [ "$backend" = "cuda" ]; then
-      [ -n "$backend_version" ] || { echo "CUDA dependencies require a backend version" >&2; exit 1; }
-      cuda_series="$(printf '%s\n' "$backend_version" | awk -F. '{ print $1 "-" $2 }')"
-      # NVIDIA runtime bases hold NCCL at the version matched to their CUDA
-      # toolkit. An unversioned explicit request would try to upgrade that hold.
-      nccl_package=libnccl2
-      nccl_installed="$(dpkg-query -W -f='${db:Status-Status} ${Version}' libnccl2 2>/dev/null || true)"
-      case "$nccl_installed" in
-        'installed '*) nccl_package="libnccl2=${nccl_installed#installed }" ;;
-      esac
-      set -- "$@" "cuda-cudart-$cuda_series" "libcublas-$cuda_series" "$nccl_package"
-    fi
+    # CUDA installs nothing. The native runtime carries its own cudart, cuBLAS,
+    # cuBLASLt, and nvJitLink closure, and the NVIDIA driver stays host-owned.
     if [ "$backend" = "rocm" ]; then
       set -- "$@" hipblas
     fi
@@ -58,9 +47,6 @@ case "$distro" in
     if [ "$backend" = "vulkan" ]; then
       set -- "$@" vulkan-icd-loader
     fi
-    if [ "$backend" = "cuda" ]; then
-      set -- "$@" cuda
-    fi
     if [ "$backend" = "rocm" ]; then
       set -- "$@" hip-runtime-amd rocm-core
     fi

docs/gpu-runbooks.md

@@ -8,7 +8,11 @@ Confirm the generated `upstream_flavor` matches an asset published by the exact
 
 ## Package or image install failure
 
-CUDA and ROCm application archives use a major backend ABI while image bases use concrete toolkit versions. Confirm the runtime bases still exist, their major matches the archive, and the native package metadata names the corresponding vendor user-space packages. Use the lean ROCm development image plus the `hipblas` package unless the application demonstrates a dependency that only the multi-gigabyte `complete` image supplies; the complete image exceeds standard hosted-runner disk during extraction. For Arch, confirm the rolling `cuda` package remains CUDA 13; if it advances to a new major, disable the row until upstream publishes a compatible archive.
+ROCm application archives use a major backend ABI while the ROCm image base uses a concrete toolkit version. Confirm the base still exists, its major matches the archive, and the package metadata names the corresponding vendor user-space packages. Use the lean ROCm development image plus the `hipblas` package unless the application demonstrates a dependency that only the multi-gigabyte `complete` image supplies; the complete image exceeds standard hosted-runner disk during extraction.
+
+CUDA rows install no vendor user-space packages and build on the plain distro base, because the native runtime carries its own cudart, cuBLAS, cuBLASLt, and nvJitLink closure. A CUDA install failure that names a missing NVIDIA package means something reintroduced a toolkit dependency, not that the base image drifted. The row's `backend_version` now only labels the archive ABI the runtime was built against; nothing installs against it. The Arch rolling `cuda` package is no longer a row prerequisite either, so its major version drifting no longer forces the row off.
+
+If a CUDA image starts up but finds no device, check `NVIDIA_VISIBLE_DEVICES` and `NVIDIA_DRIVER_CAPABILITIES` in `docker image inspect`. The `nvidia/cuda` base used to supply those and the plain base does not; the matrix derives them for NVIDIA rows and the runtime stage sets them. Empty values on a CUDA image mean the build args did not reach the image.
 
 Vulkan images require the distro Vulkan loader. A loader package failure is downstream packaging; shader/compiler failures belong to upstream because the binary and runtime bundle are already built there.
 

docs/matrix.md

@@ -26,7 +26,7 @@ an undefined mirror.
 | `arch-vulkan` | amd64 | Vulkan | pkg.tar.zst | downstream extension |
 | `arch-cuda-13.3.1` | amd64 | CUDA 13 | pkg.tar.zst | downstream extension |
 
-The Ubuntu toolkit number describes the runtime base used by the image; the upstream archive ABI is major-versioned (`cuda-12` or `cuda-13`). Arch’s rolling CUDA version can change independently while continuing to consume the upstream CUDA 13 archive. Update the row and validate installation whenever the Arch package changes major version.
+The CUDA toolkit number in a row id labels the toolkit the upstream runtime was built against; the upstream archive ABI itself is major-versioned (`cuda-12` or `cuda-13`). It no longer selects a runtime base or any installed package: CUDA rows build on the plain distro base and the native runtime supplies its own CUDA user-space closure.
 
 ## Explicit exclusion
 

docs/native-packages.md

@@ -31,7 +31,11 @@ Supported emitted formats are `.deb` for Ubuntu and `.pkg.tar.zst` for Arch. APK
 
 All variants use the package identity `mesh-llm`; backend/distro details belong in the immutable filename and description. This makes switching variants a package upgrade instead of allowing conflicting packages to own the same binary path.
 
-Native metadata declares the user-space loader dependencies needed by the selected backend. Ubuntu CUDA packages depend on the matching toolkit-series CUDA runtime, cuBLAS, and NCCL packages; Ubuntu ROCm depends on hipBLAS, which pulls its ROCm BLAS/runtime closure. The GPU vendor repository is therefore a prerequisite for installing those packages outside the configured vendor base. Host driver libraries and devices are intentionally not package dependencies.
+Native metadata declares the user-space loader dependencies needed by the selected backend. Ubuntu ROCm depends on hipBLAS, which pulls its ROCm BLAS/runtime closure, so the AMD vendor repository is a prerequisite for installing it outside the configured vendor base. Host driver libraries and devices are intentionally not package dependencies.
+
+CUDA declares no user-space toolkit dependency on any distro. Upstream packages cudart, cuBLAS, cuBLASLt, and nvJitLink into the native runtime tree and verifies that closure resolves with `LD_LIBRARY_PATH` unset, so requiring `cuda-cudart`, `libcublas`, `libnccl2`, or Arch's `cuda` would force an NVIDIA repository on users for libraries the package already installs. The NVIDIA driver stays host-owned.
+
+Because CUDA images no longer need a vendor base, they build on the plain distro runtime base. `NVIDIA_VISIBLE_DEVICES` and `NVIDIA_DRIVER_CAPABILITIES` came from the `nvidia/cuda` base before that change, and the container toolkit reads them to inject the host driver, so the matrix now derives them per row and the runtime stage sets them explicitly. Non-NVIDIA rows leave both empty, which the toolkit reads the same as unset.
 
 `scripts/native-package-qa.sh` verifies the exact filename and single-package
 invariant, writes SHA256 manifests, inspects native metadata, installs through

packaging/images.json

@@ -102,7 +102,7 @@
       "release_track": "upstream_mirrored",
       "package_base_image": "ubuntu:24.04",
       "package_format": "deb",
-      "runtime_base_image": "nvidia/cuda:12.9.2-runtime-ubuntu24.04",
+      "runtime_base_image": "ubuntu:24.04",
       "platforms": ["linux/amd64", "linux/arm64"]
     },
     {
@@ -115,7 +115,7 @@
       "release_track": "upstream_mirrored",
       "package_base_image": "ubuntu:24.04",
       "package_format": "deb",
-      "runtime_base_image": "nvidia/cuda:13.1.2-runtime-ubuntu24.04",
+      "runtime_base_image": "ubuntu:24.04",
       "platforms": ["linux/amd64", "linux/arm64"]
     },
     {

packaging/native/build-package.sh

@@ -97,11 +97,10 @@ case "$distro" in
     if [ "$backend" = "vulkan" ]; then
       depends="$depends, libvulkan1"
     fi
-    if [ "$backend" = "cuda" ]; then
-      [ -n "$backend_version" ] || { echo "CUDA package requires a backend version" >&2; exit 1; }
-      cuda_series="$(printf '%s\n' "$backend_version" | awk -F. '{ print $1 "-" $2 }')"
-      depends="$depends, cuda-cudart-$cuda_series, libcublas-$cuda_series, libnccl2"
-    fi
+    # CUDA declares no user-space toolkit dependency. The versioned runtime tree
+    # this package installs carries its own cudart, cuBLAS, cuBLASLt, and
+    # nvJitLink closure, so requiring the distro's toolkit packages would force
+    # an NVIDIA apt repository on users for libraries they already have.
     if [ "$backend" = "rocm" ]; then
       depends="$depends, hipblas"
     fi
@@ -169,9 +168,6 @@ EOF
     if [ "$backend" = "vulkan" ]; then
       printf '%s\n' 'depend = vulkan-icd-loader' >> "$root_dir/.PKGINFO"
     fi
-    if [ "$backend" = "cuda" ]; then
-      printf '%s\n' 'depend = cuda' >> "$root_dir/.PKGINFO"
-    fi
     if [ "$backend" = "rocm" ]; then
       printf '%s\n' 'depend = hip-runtime-amd' >> "$root_dir/.PKGINFO"
     fi

scripts/image-matrix.ts

@@ -91,6 +91,8 @@ export type MatrixRow = {
   package_format: string;
   runtime_base_image: string;
   runtime_base_cache_repository: string;
+  nvidia_visible_devices: string;
+  nvidia_driver_capabilities: string;
   mesh_ref: string;
   mesh_repository: string;
   mesh_version: string;
@@ -463,6 +465,9 @@ export function matrixRows(
 
       const flavor = variant.upstream_flavor as UpstreamFlavor;
       const asset = upstreamAssetName(version, targetTriple(platform), flavor);
+      // Plain distro runtime bases do not carry the NVIDIA container-toolkit
+      // variables that inject the host driver, so NVIDIA rows declare them.
+      const nvidiaRow = variant.backend === "cuda";
       rows.push({
         artifact_id: rowArtifactId,
         package_file: nativePackageFile(version, variant, arch),
@@ -486,6 +491,8 @@ export function matrixRows(
         package_format: requiredString(variant.package_format),
         runtime_base_image: requiredString(variant.runtime_base_image),
         runtime_base_cache_repository: depotRepositoryFor(config, requiredString(variant.runtime_base_image)),
+        nvidia_visible_devices: nvidiaRow ? "all" : "",
+        nvidia_driver_capabilities: nvidiaRow ? "compute,utility" : "",
         mesh_ref: meshRef,
         mesh_repository: meshRepository,
         mesh_version: version,

tests/image-matrix.test.ts

@@ -55,9 +55,21 @@ test("repository config models the supported upstream archive and packaging cont
   assert.equal(arch.package_base_cache_repository, "dockerhub-archlinux");
   assert.equal(arch.runtime_base_cache_repository, "dockerhub-archlinux");
   const cuda = rows.find((row) => row.artifact_id === "ubuntu-cuda-12.9.2-amd64")!;
-  assert.equal(cuda.runtime_base_cache_repository, "dockerhub-nvidia-cuda");
+  // The packaged runtime carries its own CUDA user-space closure, so CUDA rows
+  // run on the plain distro base and declare the container-toolkit variables
+  // the nvidia/cuda base used to supply.
+  assert.equal(cuda.runtime_base_image, "ubuntu:24.04");
+  assert.equal(cuda.runtime_base_cache_repository, "dockerhub-ubuntu");
+  for (const row of [cuda, arch]) {
+    assert.equal(row.nvidia_visible_devices, "all");
+    assert.equal(row.nvidia_driver_capabilities, "compute,utility");
+  }
   const rocm = rows.find((row) => row.artifact_id === "ubuntu-rocm-7.0-amd64")!;
   assert.equal(rocm.runtime_base_cache_repository, "dockerhub-rocm-dev-ubuntu-24-04");
+  for (const row of [cpu, rocm]) {
+    assert.equal(row.nvidia_visible_devices, "");
+    assert.equal(row.nvidia_driver_capabilities, "");
+  }
 });
 
 test("filters and disabled rows are deterministic", () => {

tests/native-package.test.ts

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
 import { spawnSync } from "node:child_process";
-import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { resolve } from "node:path";
 import { test } from "node:test";
@@ -29,3 +29,39 @@ test("package builder rejects a missing native runtime directory", (t) => {
   assert.match(result.stderr, /native runtime directory not found/);
   assert.doesNotMatch(result.stderr, /find:/);
 });
+
+test("CUDA packages declare no user-space CUDA dependency", (t) => {
+  const directory = mkdtempSync(resolve(tmpdir(), "native-package-cuda-"));
+  t.after(() => rmSync(directory, { recursive: true, force: true }));
+  const bundle = resolve(directory, "bundle");
+  mkdirSync(resolve(bundle, "native-runtimes/linux-cuda/lib"), { recursive: true });
+  writeFileSync(resolve(bundle, "mesh-llm"), "binary\n");
+  writeFileSync(resolve(bundle, "product-manifest.json"), "{}\n");
+  writeFileSync(resolve(bundle, "host-imports.json"), "{}\n");
+  writeFileSync(resolve(bundle, "native-runtimes/linux-cuda/manifest.json"), "{}\n");
+  writeFileSync(resolve(bundle, "native-runtimes/linux-cuda/lib/libllama.so"), "runtime\n");
+
+  // Capture the metadata the builder writes without needing distro tooling.
+  const stubs = resolve(directory, "stubs");
+  mkdirSync(stubs);
+  const metadata = resolve(directory, "metadata.txt");
+  // GNU-only invocations the builder makes; this test only cares about metadata.
+  writeFileSync(resolve(stubs, "du"), "#!/bin/sh\nprintf '1024\\t.\\n'\n", { mode: 0o755 });
+  writeFileSync(resolve(stubs, "touch"), "#!/bin/sh\nexit 0\n", { mode: 0o755 });
+  writeFileSync(resolve(stubs, "dpkg-deb"), `#!/bin/sh\ncat "$3/DEBIAN/control" >> ${metadata}\n:> "$4"\n`, { mode: 0o755 });
+  writeFileSync(resolve(stubs, "tar"), `#!/bin/sh\nfor arg in "$@"; do case "$arg" in -C) next=root ;; *) [ "\${next:-}" = root ] && cat "$arg/.PKGINFO" >> ${metadata} && next= ;; esac; done\n:> "$2"\n`, { mode: 0o755 });
+
+  for (const [distro, version] of [["ubuntu", "12.9.2"], ["arch", "13.3.1"]]) {
+    writeFileSync(metadata, "");
+    const result = spawnSync("sh", [
+      resolve("packaging/native/build-package.sh"), distro, "cuda", version, "amd64", "0.73.1",
+      resolve(directory, `out-${distro}`), bundle,
+    ], { encoding: "utf8", env: { ...process.env, PATH: `${stubs}:${process.env.PATH}`, SOURCE_DATE_EPOCH: "1700000000" } });
+    assert.equal(result.status, 0, result.stderr);
+    const written = readFileSync(metadata, "utf8");
+    assert.ok(written.length > 0, `${distro}: no package metadata captured`);
+    // The versioned runtime tree this package installs already carries cudart,
+    // cuBLAS, cuBLASLt, and nvJitLink. The driver is host-owned.
+    assert.doesNotMatch(written, /cudart|cublas|nccl|depend = cuda$|Depends:.*\bcuda\b/im, `${distro}: ${written}`);
+  }
+});