From 0cd727c75f1592ae88636c16cb66fe5188a4a64e Mon Sep 17 00:00:00 2001
From: Tomas Srnka <tomas.srnka@gmail.com>
Date: Wed, 22 Jul 2026 13:14:45 +0200
Subject: [PATCH 1/4] ci: probe Blacksmith kernels for nbd/KVM/module-loading
 support

Diagnostic-only workflow: dumps kernel config and device availability on
Blacksmith x64/arm64 runners and attempts to build the nbd module from
matching kernel.org source and insmod it. Establishes whether the
orchestrator unit shard and integration tests can be provisioned to run
there. Temporary; removed before any of this merges.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/blacksmith-kernel-probe.yml | 149 ++++++++++++++++++
 1 file changed, 149 insertions(+)
 create mode 100644 .github/workflows/blacksmith-kernel-probe.yml

diff --git a/.github/workflows/blacksmith-kernel-probe.yml b/.github/workflows/blacksmith-kernel-probe.yml
new file mode 100644
index 0000000000..a5b2939b68
--- /dev/null
+++ b/.github/workflows/blacksmith-kernel-probe.yml
@@ -0,0 +1,149 @@
+name: Blacksmith Kernel Probe
+
+# Temporary diagnostic workflow: establishes whether Blacksmith's custom VM
+# kernel can support the orchestrator/integration test requirements (nbd
+# module, KVM, tun, vsock, uffd, hugepages). Delete once the verdict is in.
+
+on:
+  pull_request:
+    paths:
+      - ".github/workflows/blacksmith-kernel-probe.yml"
+
+permissions:
+  contents: read
+
+jobs:
+  probe:
+    strategy:
+      fail-fast: false
+      matrix:
+        runner:
+          - blacksmith-8vcpu-ubuntu-2404
+          - blacksmith-32vcpu-ubuntu-2404
+          - blacksmith-4vcpu-ubuntu-2404-arm
+          - blacksmith-2vcpu-ubuntu-2204
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 30
+    steps:
+      - name: Kernel and device diagnostics
+        run: |
+          set -x
+          uname -a
+          cat /proc/version || true
+          echo "--- /lib/modules ---"
+          ls -la /lib/modules/ || true
+          ls /lib/modules/"$(uname -r)"/ 2>/dev/null || echo "no modules dir for running kernel"
+          find /lib/modules -name 'nbd.ko*' 2>/dev/null || true
+          grep -c . /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "no modules.builtin"
+          grep nbd /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "nbd not builtin (or no list)"
+          echo "--- kernel config ---"
+          CONFIG_SRC=""
+          if [ -e /proc/config.gz ]; then CONFIG_SRC=/proc/config.gz
+          elif [ -e "/boot/config-$(uname -r)" ]; then CONFIG_SRC="/boot/config-$(uname -r)"
+          fi
+          echo "config source: ${CONFIG_SRC:-NONE}"
+          if [ -n "$CONFIG_SRC" ]; then
+            zgrep -hE '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_SIG_ALL|MODULE_FORCE_LOAD|MODULE_COMPRESS|BLK_DEV_NBD|BLK_DEV_LOOP|TUN|VSOCKETS|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|KVM_INTEL|KVM_AMD|IKHEADERS|IKCONFIG|NF_TABLES|IP_NF_IPTABLES|NETFILTER_XTABLES|BRIDGE|VETH|FUSE_FS|OVERLAY_FS|SQUASHFS|LOCALVERSION)[ =]' "$CONFIG_SRC" 2>/dev/null \
+              || { gzip -t "$CONFIG_SRC" 2>/dev/null && zcat "$CONFIG_SRC" | grep -E '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|TUN|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|IKHEADERS)[ =]'; } || true
+          fi
+          echo "--- devices ---"
+          ls -la /dev/kvm /dev/net/tun /dev/vhost-vsock /dev/fuse /dev/nbd0 /dev/loop0 /dev/loop-control 2>&1 || true
+          echo "--- loaded modules ---"
+          lsmod | head -30 || cat /proc/modules | head -30 || true
+          echo "--- kheaders availability ---"
+          sudo modprobe kheaders 2>&1 || true
+          ls -la /sys/kernel/kheaders.tar.xz 2>&1 || true
+          echo "--- headers / build tree ---"
+          ls -la /lib/modules/"$(uname -r)"/build 2>&1 || true
+          ls /usr/src/ 2>&1 || true
+          apt-get -s install "linux-headers-$(uname -r)" 2>&1 | tail -3 || true
+          echo "--- host features ---"
+          nproc; free -h; df -h / /tmp
+          sudo sysctl vm.unprivileged_userfaultfd || true
+          mount | grep -iE 'hugetlbfs|cgroup2' || true
+          docker info --format '{{.Driver}} kernel={{.KernelVersion}}' 2>/dev/null || true
+
+      - name: Attempt nbd module build from kernel.org source
+        id: build
+        continue-on-error: true
+        run: |
+          set -euxo pipefail
+          T0=$(date +%s)
+          KVER="$(uname -r)"
+          BASE="${KVER%%-*}"
+
+          # Bail out early if modprobe already works (e.g. image got fixed)
+          if sudo modprobe nbd nbds_max=256 2>/dev/null; then
+            echo "modprobe worked natively, nothing to build"
+            exit 0
+          fi
+
+          # Kernel config is required to build a compatible module
+          if [ -e /proc/config.gz ]; then
+            CONFIG_CMD="zcat /proc/config.gz"
+          elif [ -e "/boot/config-$KVER" ]; then
+            CONFIG_CMD="cat /boot/config-$KVER"
+          else
+            echo "::error::no kernel config available (no /proc/config.gz, no /boot/config)"
+            exit 1
+          fi
+
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev dwarves
+
+          cd /tmp
+          curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz"
+          tar -xf "linux-${BASE}.tar.xz"
+          cd "linux-${BASE}"
+
+          eval "$CONFIG_CMD" > .config
+          # Force nbd to build as a module regardless of the shipped config
+          ./scripts/config -m BLK_DEV_NBD
+          # A signing key we don't have; disable so modpost doesn't try to sign
+          ./scripts/config --disable MODULE_SIG_ALL || true
+          ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true
+          ./scripts/config --set-str MODULE_SIG_KEY "" || true
+          make olddefconfig
+          echo "--- effective module-relevant config ---"
+          grep -E '^CONFIG_(MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|LOCALVERSION)[ =]' .config || true
+
+          T1=$(date +%s); echo "setup took $((T1-T0))s"
+          make -j"$(nproc)" modules_prepare
+          T2=$(date +%s); echo "modules_prepare took $((T2-T1))s"
+          make -j"$(nproc)" M=drivers/block modules
+          T3=$(date +%s); echo "module build took $((T3-T2))s"
+
+          modinfo drivers/block/nbd.ko | head -20
+          echo "--- vermagic comparison ---"
+          modinfo -F vermagic drivers/block/nbd.ko || true
+          echo "running kernel: $KVER"
+
+          if ! sudo insmod drivers/block/nbd.ko nbds_max=256; then
+            echo "plain insmod failed, dmesg:"
+            sudo dmesg | tail -20
+            # Retry via modprobe --force (strips vermagic/version checks;
+            # needs CONFIG_MODULE_FORCE_LOAD=y in the running kernel)
+            sudo mkdir -p "/lib/modules/$KVER/kernel/drivers/block"
+            sudo cp drivers/block/nbd.ko "/lib/modules/$KVER/kernel/drivers/block/"
+            sudo depmod -a || true
+            sudo modprobe --force nbd nbds_max=256 || {
+              echo "forced modprobe also failed, dmesg:"
+              sudo dmesg | tail -20
+              exit 1
+            }
+          fi
+          T4=$(date +%s); echo "TOTAL nbd provisioning: $((T4-T0))s"
+
+      - name: Verdict
+        run: |
+          set -x
+          echo "=== VERDICT for ${{ matrix.runner }} ==="
+          NBD_OK=false; KVM_OK=false; TUN_OK=false; VSOCK_OK=false
+          [ -b /dev/nbd0 ] && [ -b /dev/nbd255 ] && NBD_OK=true
+          [ -e /dev/kvm ] && KVM_OK=true
+          [ -e /dev/net/tun ] && TUN_OK=true
+          [ -e /dev/vhost-vsock ] && VSOCK_OK=true
+          sudo dmesg | grep -iE 'nbd|taint' | tail -5 || true
+          echo "NBD=$NBD_OK KVM=$KVM_OK TUN=$TUN_OK VSOCK=$VSOCK_OK"
+          # nbd is the make-or-break requirement on every runner
+          $NBD_OK

From 50bd7a15be5b5433339214b76a2d99f31a1d1c0a Mon Sep 17 00:00:00 2001
From: Tomas Srnka <tomas.srnka@gmail.com>
Date: Wed, 22 Jul 2026 13:20:57 +0200
Subject: [PATCH 2/4] ci: probe v2 - netlink nbd creation on x64,
 KBUILD_MODPOST_WARN build on arm64

v1 verdicts: x64 kernel has nbd BUILTIN (CONFIG_BLK_DEV_NBD=y, /dev/nbd0
exists, /dev/kvm + tun + uffd all present) so no module is needed - only
more device nodes, which nbd's netlink interface creates on demand. arm64
has no nbd at all, so it needs the source-built module; v1's modpost
failed on undefined core symbols (no Module.symvers), fixed with
KBUILD_MODPOST_WARN=1. Also add swapfile smoke test (init-client.sh
swapon) and /proc/cmdline + device-count inventory.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/workflows/blacksmith-kernel-probe.yml | 190 +++++++++---------
 1 file changed, 93 insertions(+), 97 deletions(-)

diff --git a/.github/workflows/blacksmith-kernel-probe.yml b/.github/workflows/blacksmith-kernel-probe.yml
index a5b2939b68..343a785ca2 100644
--- a/.github/workflows/blacksmith-kernel-probe.yml
+++ b/.github/workflows/blacksmith-kernel-probe.yml
@@ -2,7 +2,12 @@ name: Blacksmith Kernel Probe
 
 # Temporary diagnostic workflow: establishes whether Blacksmith's custom VM
 # kernel can support the orchestrator/integration test requirements (nbd
-# module, KVM, tun, vsock, uffd, hugepages). Delete once the verdict is in.
+# devices, KVM, tun, uffd, hugepages, swap). Delete once the verdict is in.
+#
+# Probe v1 findings: x64 has CONFIG_BLK_DEV_NBD=y (builtin, /dev/nbd0 exists,
+# count unknown) + /dev/kvm + tun + uffd; arm64 has NO nbd at all and no
+# /dev/kvm. So: x64 tests netlink-based nbd device creation; arm64 builds
+# nbd.ko from source (KBUILD_MODPOST_WARN=1 for the missing Module.symvers).
 
 on:
   pull_request:
@@ -17,11 +22,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runner:
-          - blacksmith-8vcpu-ubuntu-2404
-          - blacksmith-32vcpu-ubuntu-2404
-          - blacksmith-4vcpu-ubuntu-2404-arm
-          - blacksmith-2vcpu-ubuntu-2204
+        include:
+          - runner: blacksmith-8vcpu-ubuntu-2404
+            arch: x64
+          - runner: blacksmith-32vcpu-ubuntu-2404
+            arch: x64
+          - runner: blacksmith-4vcpu-ubuntu-2404-arm
+            arch: arm64
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 30
     steps:
@@ -29,121 +36,110 @@ jobs:
         run: |
           set -x
           uname -a
-          cat /proc/version || true
-          echo "--- /lib/modules ---"
-          ls -la /lib/modules/ || true
-          ls /lib/modules/"$(uname -r)"/ 2>/dev/null || echo "no modules dir for running kernel"
-          find /lib/modules -name 'nbd.ko*' 2>/dev/null || true
-          grep -c . /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "no modules.builtin"
-          grep nbd /lib/modules/"$(uname -r)"/modules.builtin 2>/dev/null || echo "nbd not builtin (or no list)"
-          echo "--- kernel config ---"
-          CONFIG_SRC=""
-          if [ -e /proc/config.gz ]; then CONFIG_SRC=/proc/config.gz
-          elif [ -e "/boot/config-$(uname -r)" ]; then CONFIG_SRC="/boot/config-$(uname -r)"
-          fi
-          echo "config source: ${CONFIG_SRC:-NONE}"
-          if [ -n "$CONFIG_SRC" ]; then
-            zgrep -hE '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_SIG_ALL|MODULE_FORCE_LOAD|MODULE_COMPRESS|BLK_DEV_NBD|BLK_DEV_LOOP|TUN|VSOCKETS|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|KVM_INTEL|KVM_AMD|IKHEADERS|IKCONFIG|NF_TABLES|IP_NF_IPTABLES|NETFILTER_XTABLES|BRIDGE|VETH|FUSE_FS|OVERLAY_FS|SQUASHFS|LOCALVERSION)[ =]' "$CONFIG_SRC" 2>/dev/null \
-              || { gzip -t "$CONFIG_SRC" 2>/dev/null && zcat "$CONFIG_SRC" | grep -E '^CONFIG_(MODULES|MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|TUN|VHOST_VSOCK|USERFAULTFD|HUGETLBFS|KVM|IKHEADERS)[ =]'; } || true
-          fi
+          cat /proc/cmdline || true
+          echo "--- nbd device inventory ---"
+          ls /dev/nbd* 2>/dev/null | wc -l
+          ls /dev/nbd* 2>/dev/null | sort -V | tail -3 || true
+          echo "--- kernel config (incl. 'not set' lines) ---"
+          zcat /proc/config.gz | grep -E '(MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|MODULE_COMPRESS|BLK_DEV_NBD|VSOCKETS|VHOST_VSOCK|SWAP|MEMORY_HOTPLUG)[ =]' || true
           echo "--- devices ---"
-          ls -la /dev/kvm /dev/net/tun /dev/vhost-vsock /dev/fuse /dev/nbd0 /dev/loop0 /dev/loop-control 2>&1 || true
-          echo "--- loaded modules ---"
-          lsmod | head -30 || cat /proc/modules | head -30 || true
-          echo "--- kheaders availability ---"
-          sudo modprobe kheaders 2>&1 || true
-          ls -la /sys/kernel/kheaders.tar.xz 2>&1 || true
-          echo "--- headers / build tree ---"
-          ls -la /lib/modules/"$(uname -r)"/build 2>&1 || true
-          ls /usr/src/ 2>&1 || true
-          apt-get -s install "linux-headers-$(uname -r)" 2>&1 | tail -3 || true
-          echo "--- host features ---"
-          nproc; free -h; df -h / /tmp
-          sudo sysctl vm.unprivileged_userfaultfd || true
-          mount | grep -iE 'hugetlbfs|cgroup2' || true
-          docker info --format '{{.Driver}} kernel={{.KernelVersion}}' 2>/dev/null || true
+          ls -la /dev/kvm /dev/net/tun /dev/vhost-vsock 2>&1 || true
+          echo "--- swap support ---"
+          cat /proc/swaps || true
+          echo "--- filesystem of / (fallocate swapfile support) ---"
+          df -T / | tail -1
 
-      - name: Attempt nbd module build from kernel.org source
-        id: build
+      - name: Swapfile smoke test (init-client.sh does swapon)
         continue-on-error: true
         run: |
           set -euxo pipefail
-          T0=$(date +%s)
-          KVER="$(uname -r)"
-          BASE="${KVER%%-*}"
+          sudo fallocate -l 512M /swapfile-probe
+          sudo chmod 600 /swapfile-probe
+          sudo mkswap /swapfile-probe
+          sudo swapon /swapfile-probe
+          cat /proc/swaps
+          sudo swapoff /swapfile-probe
+          echo "SWAP_OK"
 
-          # Bail out early if modprobe already works (e.g. image got fixed)
-          if sudo modprobe nbd nbds_max=256 2>/dev/null; then
-            echo "modprobe worked natively, nothing to build"
-            exit 0
-          fi
+      - name: "x64: netlink nbd device creation test"
+        if: matrix.arch == 'x64'
+        run: |
+          set -euxo pipefail
+          sudo apt-get update -qq && sudo apt-get install -y -qq nbd-server nbd-client
+          # nbd-server package start may fail as a service; run manually
+          dd if=/dev/zero of=/tmp/nbd-export.img bs=1M count=8
+          cat > /tmp/nbd.conf <<'EOF'
+          [generic]
+            allowlist = false
+          [probe]
+            exportname = /tmp/nbd-export.img
+          EOF
+          sudo pkill nbd-server || true
+          sudo nbd-server -C /tmp/nbd.conf
+          sleep 1
 
-          # Kernel config is required to build a compatible module
-          if [ -e /proc/config.gz ]; then
-            CONFIG_CMD="zcat /proc/config.gz"
-          elif [ -e "/boot/config-$KVER" ]; then
-            CONFIG_CMD="cat /boot/config-$KVER"
-          else
-            echo "::error::no kernel config available (no /proc/config.gz, no /boot/config)"
-            exit 1
-          fi
+          echo "--- devices before ---"; N_BEFORE=$(ls /dev/nbd* | wc -l)
 
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev dwarves
+          # Ask for an index that does not exist yet; netlink-capable
+          # nbd-client makes the kernel create it on demand.
+          sudo nbd-client -N probe 127.0.0.1 /dev/nbd200
+          ls -la /dev/nbd200
+          sudo nbd-client -d /dev/nbd200
+
+          echo "--- bulk creation timing (50 devices) ---"
+          T0=$(date +%s)
+          for i in $(seq 300 349); do
+            sudo nbd-client -N probe 127.0.0.1 "/dev/nbd$i" >/dev/null 2>&1
+            sudo nbd-client -d "/dev/nbd$i" >/dev/null 2>&1
+          done
+          T1=$(date +%s)
+          echo "50 devices in $((T1-T0))s"
+          N_AFTER=$(ls /dev/nbd* | wc -l)
+          echo "device count: $N_BEFORE -> $N_AFTER"
+          [ -b /dev/nbd349 ]
 
+      - name: "arm64: build nbd.ko from source and insmod"
+        if: matrix.arch == 'arm64'
+        run: |
+          set -euxo pipefail
+          T0=$(date +%s)
+          KVER="$(uname -r)"
+          BASE="${KVER%%-*}"
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev
           cd /tmp
           curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz"
           tar -xf "linux-${BASE}.tar.xz"
           cd "linux-${BASE}"
-
-          eval "$CONFIG_CMD" > .config
-          # Force nbd to build as a module regardless of the shipped config
+          zcat /proc/config.gz > .config
           ./scripts/config -m BLK_DEV_NBD
-          # A signing key we don't have; disable so modpost doesn't try to sign
           ./scripts/config --disable MODULE_SIG_ALL || true
           ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true
-          ./scripts/config --set-str MODULE_SIG_KEY "" || true
           make olddefconfig
-          echo "--- effective module-relevant config ---"
-          grep -E '^CONFIG_(MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|BLK_DEV_NBD|LOCALVERSION)[ =]' .config || true
-
-          T1=$(date +%s); echo "setup took $((T1-T0))s"
+          grep -E '(MODVERSIONS|MODULE_SIG|BLK_DEV_NBD|LOCALVERSION)[ =]' .config || true
           make -j"$(nproc)" modules_prepare
-          T2=$(date +%s); echo "modules_prepare took $((T2-T1))s"
-          make -j"$(nproc)" M=drivers/block modules
-          T3=$(date +%s); echo "module build took $((T3-T2))s"
-
-          modinfo drivers/block/nbd.ko | head -20
-          echo "--- vermagic comparison ---"
-          modinfo -F vermagic drivers/block/nbd.ko || true
+          # No Module.symvers for the running kernel: modpost cannot resolve
+          # core symbols and errors out; WARN=1 downgrades that. Runtime
+          # symbol resolution happens in the kernel at insmod.
+          make -j"$(nproc)" KBUILD_MODPOST_WARN=1 M=drivers/block modules
+          modinfo -F vermagic drivers/block/nbd.ko
           echo "running kernel: $KVER"
-
           if ! sudo insmod drivers/block/nbd.ko nbds_max=256; then
-            echo "plain insmod failed, dmesg:"
-            sudo dmesg | tail -20
-            # Retry via modprobe --force (strips vermagic/version checks;
-            # needs CONFIG_MODULE_FORCE_LOAD=y in the running kernel)
-            sudo mkdir -p "/lib/modules/$KVER/kernel/drivers/block"
-            sudo cp drivers/block/nbd.ko "/lib/modules/$KVER/kernel/drivers/block/"
-            sudo depmod -a || true
-            sudo modprobe --force nbd nbds_max=256 || {
-              echo "forced modprobe also failed, dmesg:"
-              sudo dmesg | tail -20
-              exit 1
-            }
+            echo "insmod failed, dmesg:"
+            sudo dmesg | tail -15
+            exit 1
           fi
-          T4=$(date +%s); echo "TOTAL nbd provisioning: $((T4-T0))s"
+          T1=$(date +%s)
+          echo "TOTAL arm nbd provisioning: $((T1-T0))s"
+          ls /dev/nbd* | wc -l
+          [ -b /dev/nbd0 ] && [ -b /dev/nbd255 ]
+          # size the cacheable artifact
+          ls -la drivers/block/nbd.ko
 
       - name: Verdict
         run: |
           set -x
           echo "=== VERDICT for ${{ matrix.runner }} ==="
-          NBD_OK=false; KVM_OK=false; TUN_OK=false; VSOCK_OK=false
-          [ -b /dev/nbd0 ] && [ -b /dev/nbd255 ] && NBD_OK=true
-          [ -e /dev/kvm ] && KVM_OK=true
-          [ -e /dev/net/tun ] && TUN_OK=true
-          [ -e /dev/vhost-vsock ] && VSOCK_OK=true
+          ls /dev/nbd* 2>/dev/null | wc -l
+          [ -e /dev/kvm ] && echo KVM=yes || echo KVM=no
           sudo dmesg | grep -iE 'nbd|taint' | tail -5 || true
-          echo "NBD=$NBD_OK KVM=$KVM_OK TUN=$TUN_OK VSOCK=$VSOCK_OK"
-          # nbd is the make-or-break requirement on every runner
-          $NBD_OK

From c6e2820bf499833d7d8e637469d59c22b231a936 Mon Sep 17 00:00:00 2001
From: Tomas Srnka <tomas.srnka@gmail.com>
Date: Wed, 22 Jul 2026 13:28:49 +0200
Subject: [PATCH 3/4] ci: run orchestrator shards and integration tests on
 Blacksmith

Probe results (see blacksmith-kernel-probe.yml runs): Blacksmith x64
kernels ship nbd BUILT IN (16 devices) plus /dev/kvm, tun, uffd,
hugepages, swap and cgroup2 - everything the orchestrator needs; the
device pool clamps to nbds_max so 16 devices just caps concurrency
(unit tests use ~8, integration parallelism is -parallel=4). arm64
kernels ship no nbd at all, but a module built from matching kernel.org
source with the running kernel's /proc/config.gz insmods cleanly (36s
cold, cached per kernel release thereafter).

- new ensure-nbd composite action: driver-present / modprobe /
  linux-modules-extra / build-from-source, in that order, so the same
  workflows keep working on GitHub-hosted runners
- orchestrator unit shards: infra-tests -> blacksmith-8vcpu,
  infra-runner-arm -> blacksmith-4vcpu-arm
- integration tests: infra-tests -> blacksmith-32vcpu
- init-client.sh: skip modprobe when the driver is built in

Verification criteria: TestSmokeAllFCVersions must RUN (not skip) on
x64, no new environment-gated skips, no NoFreeSlotsError in integration
service logs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/actions/ensure-nbd/action.yml    | 112 +++++++++++++++++++++++
 .github/actions/host-init/init-client.sh |  11 ++-
 .github/workflows/integration_tests.yml  |   8 +-
 .github/workflows/pr-tests-arm64.yml     |  23 +++--
 .github/workflows/pr-tests.yml           |  23 ++---
 .github/workflows/pull-request.yml       |   1 +
 6 files changed, 152 insertions(+), 26 deletions(-)
 create mode 100644 .github/actions/ensure-nbd/action.yml

diff --git a/.github/actions/ensure-nbd/action.yml b/.github/actions/ensure-nbd/action.yml
new file mode 100644
index 0000000000..73271cfdca
--- /dev/null
+++ b/.github/actions/ensure-nbd/action.yml
@@ -0,0 +1,112 @@
+name: "Ensure NBD devices"
+description: |
+  Makes the nbd block driver available with device nodes for the
+  orchestrator's DevicePool (which sizes itself from
+  /sys/module/nbd/parameters/nbds_max and refuses to start without it).
+  Handles, in order:
+    1. driver already present (loaded module OR built into the kernel,
+       e.g. Blacksmith x64 runners) - nothing to do, the existing device
+       count applies;
+    2. stock kernel with the module shipped - plain modprobe;
+    3. stock Ubuntu kernel with nbd in linux-modules-extra (GitHub-hosted
+       arm64 runners) - apt install, then modprobe;
+    4. custom kernel without an nbd module at all (Blacksmith arm64
+       runners) - build nbd.ko from the matching kernel.org source using
+       the running kernel's /proc/config.gz, insmod it. The built .ko is
+       cached per kernel release (~36s to build cold, instant when cached).
+
+inputs:
+  nbds-max:
+    description: "Device count to request when loading the module (no effect when the driver is built in)"
+    required: false
+    default: "4096"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Detect nbd provisioning strategy
+      id: detect
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [ -r /sys/module/nbd/parameters/nbds_max ]; then
+          echo "nbd driver already present (nbds_max=$(cat /sys/module/nbd/parameters/nbds_max))"
+          echo "strategy=none" >> "$GITHUB_OUTPUT"
+        elif sudo modprobe nbd nbds_max='${{ inputs.nbds-max }}' 2>/dev/null; then
+          echo "strategy=modprobe" >> "$GITHUB_OUTPUT"
+        elif apt-get -s install "linux-modules-extra-$(uname -r)" >/dev/null 2>&1; then
+          echo "strategy=apt" >> "$GITHUB_OUTPUT"
+        else
+          echo "strategy=build" >> "$GITHUB_OUTPUT"
+        fi
+        echo "kver=$(uname -r)" >> "$GITHUB_OUTPUT"
+
+    - name: Install nbd module from linux-modules-extra
+      if: steps.detect.outputs.strategy == 'apt'
+      shell: bash
+      run: |
+        set -euo pipefail
+        sudo apt-get update
+        sudo apt-get install -y "linux-modules-extra-$(uname -r)"
+        sudo modprobe nbd nbds_max='${{ inputs.nbds-max }}'
+
+    - name: Restore cached nbd.ko
+      if: steps.detect.outputs.strategy == 'build'
+      uses: actions/cache@v5
+      with:
+        path: /tmp/nbd-ko
+        key: nbd-ko-${{ runner.arch }}-${{ steps.detect.outputs.kver }}
+
+    - name: Build nbd.ko from kernel source and insmod
+      if: steps.detect.outputs.strategy == 'build'
+      shell: bash
+      run: |
+        set -euo pipefail
+        KVER="$(uname -r)"
+        if [ ! -f /tmp/nbd-ko/nbd.ko ]; then
+          if [ ! -e /proc/config.gz ]; then
+            echo "::error::kernel $KVER has no nbd module and no /proc/config.gz to build one from"
+            exit 1
+          fi
+          sudo apt-get update -qq
+          sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev
+          BASE="${KVER%%-*}"
+          BUILD_DIR="$(mktemp -d)"
+          cd "$BUILD_DIR"
+          curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz"
+          tar -xf "linux-${BASE}.tar.xz"
+          cd "linux-${BASE}"
+          zcat /proc/config.gz > .config
+          ./scripts/config -m BLK_DEV_NBD
+          # We cannot sign with the kernel's key; the running kernels this
+          # path targets have CONFIG_MODULE_SIG disabled.
+          ./scripts/config --disable MODULE_SIG_ALL || true
+          ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true
+          make olddefconfig
+          make -j"$(nproc)" modules_prepare
+          # Without the running kernel's Module.symvers, modpost cannot
+          # resolve core symbols and errors out; WARN=1 downgrades that.
+          # Actual symbol resolution happens in the kernel at insmod.
+          make -j"$(nproc)" KBUILD_MODPOST_WARN=1 M=drivers/block modules
+          mkdir -p /tmp/nbd-ko
+          cp drivers/block/nbd.ko /tmp/nbd-ko/
+        fi
+        sudo insmod /tmp/nbd-ko/nbd.ko nbds_max='${{ inputs.nbds-max }}' || {
+          echo "insmod failed, dmesg:"
+          sudo dmesg | tail -15
+          exit 1
+        }
+
+    - name: Verify devices and quiet udev
+      shell: bash
+      run: |
+        set -euo pipefail
+        if [ ! -r /sys/module/nbd/parameters/nbds_max ]; then
+          echo "::error::nbd driver is not available after provisioning"
+          exit 1
+        fi
+        echo "effective nbds_max=$(cat /sys/module/nbd/parameters/nbds_max)"
+        # Disable inotify watching of change events for NBD devices
+        echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules
+        sudo udevadm control --reload-rules || true
+        sudo udevadm trigger || true
diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh
index 57c4946154..0584e7b7d2 100644
--- a/.github/actions/host-init/init-client.sh
+++ b/.github/actions/host-init/init-client.sh
@@ -65,8 +65,15 @@ EOH
 sudo udevadm control --reload-rules
 sudo udevadm trigger
 
-# Load the nbd module with 4096 devices
-sudo modprobe nbd nbds_max=4096
+# Load the nbd module with 4096 devices. Kernels that build nbd in
+# (e.g. Blacksmith runners) have no module to load - the parameter file
+# already exists and the built-in device count applies; the orchestrator's
+# device pool clamps to it.
+if [ -r /sys/module/nbd/parameters/nbds_max ]; then
+  echo "nbd driver already present (nbds_max=$(cat /sys/module/nbd/parameters/nbds_max))"
+else
+  sudo modprobe nbd nbds_max=4096
+fi
 
 # Create the directory for the fc mounts
 mkdir -p /fc-vm
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index 6d00df773b..a84d82118a 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -16,8 +16,14 @@ on:
       CODECOV_TOKEN: { required: false }
 jobs:
   run:
+    # Blacksmith x64 kernels ship nbd built in (16 devices; init-client.sh
+    # skips its modprobe in that case) and expose /dev/kvm for Firecracker.
+    # The orchestrator's NBD pool clamps to the available device count, so
+    # 16 caps concurrent sandboxes - watch for NoFreeSlotsError ("no free
+    # slots") in service logs if the suite grows more parallel.
+    # 32vcpu matches the 32-core infra-tests runner this previously ran on.
     if: ${{ inputs.run-tests == true }}
-    runs-on: infra-tests
+    runs-on: blacksmith-32vcpu-ubuntu-2404
     timeout-minutes: 30
     permissions:
       contents: read
diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml
index c5ebf44824..73273abbeb 100644
--- a/.github/workflows/pr-tests-arm64.yml
+++ b/.github/workflows/pr-tests-arm64.yml
@@ -73,8 +73,11 @@ jobs:
   arm64-unit-tests:
     name: Run ARM64 test shards
     if: ${{ inputs.run-tests == true }}
-    # Shards that only need Docker run on Blacksmith arm64. The orchestrator
-    # shard stays on infra-runner-arm: it needs modprobe nbd.
+    # All shards run on Blacksmith arm64. The orchestrator shard needs the
+    # nbd driver, which the Blacksmith arm64 kernel does not ship - the
+    # ensure-nbd action builds it from source (cached per kernel release).
+    # There is no /dev/kvm on any arm64 runner (GitHub's included), so the
+    # KVM smoketest skips there either way.
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 45
     env:
@@ -120,7 +123,7 @@ jobs:
             flag: arm64-orchestrator
             test_path: ./...
             sudo: true
-            runner: infra-runner-arm
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
           - package: packages/shared
             flag: arm64-shared
             test_path: ./pkg/...
@@ -176,16 +179,12 @@ jobs:
           sudo mkdir -p /mnt/hugepages
           sudo mount -t hugetlbfs none /mnt/hugepages
           echo 256 | sudo tee /proc/sys/vm/nr_hugepages
+        if: matrix.package == 'packages/orchestrator'
 
-          # Install extra kernel modules (nbd is not in base modules on GitHub-hosted runners)
-          sudo apt-get update
-          sudo apt-get install -y linux-modules-extra-$(uname -r)
-          sudo modprobe nbd nbds_max=256
-
-          # Disable inotify watching of change events for NBD devices
-          echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger
+      - name: Ensure NBD devices
+        uses: ./.github/actions/ensure-nbd
+        with:
+          nbds-max: "256"
         if: matrix.package == 'packages/orchestrator'
 
       - name: Install gotestsum
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
index 395e7a44c7..61171eef60 100644
--- a/.github/workflows/pr-tests.yml
+++ b/.github/workflows/pr-tests.yml
@@ -18,8 +18,10 @@ jobs:
   run-tests:
     name: Run unit test shards
     if: ${{ inputs.run-tests == true }}
-    # Shards that only need Docker run on Blacksmith. The orchestrator shard
-    # stays on infra-tests: it needs modprobe nbd and /dev/kvm (smoketest).
+    # All shards run on Blacksmith. The orchestrator shard needs the nbd
+    # driver (built into the Blacksmith x64 kernel; see ensure-nbd action)
+    # and /dev/kvm (present on Blacksmith x64 - verify the smoketest RUNS,
+    # it skips silently when /dev/kvm is missing).
     runs-on: ${{ matrix.runner }}
     env:
       GIN_MODE: test
@@ -60,11 +62,13 @@ jobs:
             test_path: ./...
             sudo: true
             runner: blacksmith-4vcpu-ubuntu-2404
+          # 8vcpu: this shard boots Firecracker VMs (smoketest); it ran on a
+          # 32-core infra-tests runner before - bump if the 20m timeout nears.
           - package: packages/orchestrator
             flag: unit-orchestrator
             test_path: ./...
             sudo: true
-            runner: infra-tests
+            runner: blacksmith-8vcpu-ubuntu-2404
           - package: packages/shared
             flag: unit-shared
             test_path: ./pkg/...
@@ -120,15 +124,12 @@ jobs:
           sudo mkdir -p /mnt/hugepages
           sudo mount -t hugetlbfs none /mnt/hugepages
           echo 2000 | sudo tee /proc/sys/vm/nr_hugepages
+        if: matrix.package == 'packages/orchestrator'
 
-          # Enable NBD
-          sudo modprobe nbd nbds_max=256
-
-          # Disable inotify watching of change events for NBD devices
-          echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger
-
+      - name: Ensure NBD devices
+        uses: ./.github/actions/ensure-nbd
+        with:
+          nbds-max: "256"
         if: matrix.package == 'packages/orchestrator'
 
       - name: Install gotestsum
diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml
index e6bbbd2039..b1de6c4789 100644
--- a/.github/workflows/pull-request.yml
+++ b/.github/workflows/pull-request.yml
@@ -44,6 +44,7 @@ jobs:
               - '.github/workflows/pr-tests.yml'
               - '.github/workflows/pr-tests-arm64.yml'
               - '.github/actions/go-setup-cache/**'
+              - '.github/actions/ensure-nbd/**'
             # Like test-inputs, but scoped to only the .github paths that affect
             # the integration harness (its workflow + composite actions) instead
             # of all of .github/**. This keeps unrelated CI edits (e.g. other

From 3b2bb7fcd04f931965ede03a92d5b665f16caae8 Mon Sep 17 00:00:00 2001
From: Tomas Srnka <tomas.srnka@gmail.com>
Date: Wed, 22 Jul 2026 13:40:51 +0200
Subject: [PATCH 4/4] ci: land Blacksmith provisioning groundwork, keep KVM
 jobs on GitHub

Full-suite run on Blacksmith found the real blocker: their 6.6.141
kernel predates UFFD_FEATURE_WP_ASYNC (Linux >= 6.7), which the uffd
memory backend and FC dirty-page tracking hard-require. 208 uffd test
failures (UFFDIO_API EINVAL) and FC snapshot loads failing with 'Failed
to UFFD object' on both arches. Not provisionable from CI - needs a
Blacksmith kernel upgrade.

Keep the groundwork (no behavior change on GitHub runners):
- ensure-nbd action replaces inline modprobe/modules-extra setup in the
  unit-test workflows; it also handles builtin-nbd and no-module kernels
- init-client.sh tolerates built-in nbd and pre-existing active swap
- runner labels reverted to infra-tests / infra-runner-arm; comments
  document the exact kernel blocker and the revisit condition

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 .github/actions/host-init/init-client.sh      |  21 ++-
 .github/workflows/blacksmith-kernel-probe.yml | 145 ------------------
 .github/workflows/integration_tests.yml       |  14 +-
 .github/workflows/pr-tests-arm64.yml          |  13 +-
 .github/workflows/pr-tests.yml                |  13 +-
 5 files changed, 34 insertions(+), 172 deletions(-)
 delete mode 100644 .github/workflows/blacksmith-kernel-probe.yml

diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh
index 0584e7b7d2..cf5da687c7 100644
--- a/.github/actions/host-init/init-client.sh
+++ b/.github/actions/host-init/init-client.sh
@@ -17,15 +17,22 @@ sudo mkdir -p /orchestrator/sandbox
 sudo mkdir -p /orchestrator/template
 sudo mkdir -p /orchestrator/build
 
-# Add swapfile
+# Add swapfile. Some runner images (e.g. Blacksmith) already boot with an
+# active /swapfile; fallocate on an in-use swap file fails with ETXTBSY,
+# so only create one when no swap is active yet.
 SWAPFILE="/swapfile"
-sudo fallocate -l 1G $SWAPFILE
-sudo chmod 600 $SWAPFILE
-sudo mkswap $SWAPFILE
-sudo swapon $SWAPFILE
+if grep -q '^\S' /proc/swaps 2>/dev/null && [ "$(wc -l < /proc/swaps)" -gt 1 ]; then
+  echo "swap already active, skipping swapfile creation:"
+  cat /proc/swaps
+else
+  sudo fallocate -l 1G $SWAPFILE
+  sudo chmod 600 $SWAPFILE
+  sudo mkswap $SWAPFILE
+  sudo swapon $SWAPFILE
 
-# Make swapfile persistent
-echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab
+  # Make swapfile persistent
+  echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab
+fi
 
 # Set swap settings
 sudo sysctl vm.swappiness=10
diff --git a/.github/workflows/blacksmith-kernel-probe.yml b/.github/workflows/blacksmith-kernel-probe.yml
deleted file mode 100644
index 343a785ca2..0000000000
--- a/.github/workflows/blacksmith-kernel-probe.yml
+++ /dev/null
@@ -1,145 +0,0 @@
-name: Blacksmith Kernel Probe
-
-# Temporary diagnostic workflow: establishes whether Blacksmith's custom VM
-# kernel can support the orchestrator/integration test requirements (nbd
-# devices, KVM, tun, uffd, hugepages, swap). Delete once the verdict is in.
-#
-# Probe v1 findings: x64 has CONFIG_BLK_DEV_NBD=y (builtin, /dev/nbd0 exists,
-# count unknown) + /dev/kvm + tun + uffd; arm64 has NO nbd at all and no
-# /dev/kvm. So: x64 tests netlink-based nbd device creation; arm64 builds
-# nbd.ko from source (KBUILD_MODPOST_WARN=1 for the missing Module.symvers).
-
-on:
-  pull_request:
-    paths:
-      - ".github/workflows/blacksmith-kernel-probe.yml"
-
-permissions:
-  contents: read
-
-jobs:
-  probe:
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - runner: blacksmith-8vcpu-ubuntu-2404
-            arch: x64
-          - runner: blacksmith-32vcpu-ubuntu-2404
-            arch: x64
-          - runner: blacksmith-4vcpu-ubuntu-2404-arm
-            arch: arm64
-    runs-on: ${{ matrix.runner }}
-    timeout-minutes: 30
-    steps:
-      - name: Kernel and device diagnostics
-        run: |
-          set -x
-          uname -a
-          cat /proc/cmdline || true
-          echo "--- nbd device inventory ---"
-          ls /dev/nbd* 2>/dev/null | wc -l
-          ls /dev/nbd* 2>/dev/null | sort -V | tail -3 || true
-          echo "--- kernel config (incl. 'not set' lines) ---"
-          zcat /proc/config.gz | grep -E '(MODVERSIONS|MODULE_SIG|MODULE_SIG_FORCE|MODULE_FORCE_LOAD|MODULE_COMPRESS|BLK_DEV_NBD|VSOCKETS|VHOST_VSOCK|SWAP|MEMORY_HOTPLUG)[ =]' || true
-          echo "--- devices ---"
-          ls -la /dev/kvm /dev/net/tun /dev/vhost-vsock 2>&1 || true
-          echo "--- swap support ---"
-          cat /proc/swaps || true
-          echo "--- filesystem of / (fallocate swapfile support) ---"
-          df -T / | tail -1
-
-      - name: Swapfile smoke test (init-client.sh does swapon)
-        continue-on-error: true
-        run: |
-          set -euxo pipefail
-          sudo fallocate -l 512M /swapfile-probe
-          sudo chmod 600 /swapfile-probe
-          sudo mkswap /swapfile-probe
-          sudo swapon /swapfile-probe
-          cat /proc/swaps
-          sudo swapoff /swapfile-probe
-          echo "SWAP_OK"
-
-      - name: "x64: netlink nbd device creation test"
-        if: matrix.arch == 'x64'
-        run: |
-          set -euxo pipefail
-          sudo apt-get update -qq && sudo apt-get install -y -qq nbd-server nbd-client
-          # nbd-server package start may fail as a service; run manually
-          dd if=/dev/zero of=/tmp/nbd-export.img bs=1M count=8
-          cat > /tmp/nbd.conf <<'EOF'
-          [generic]
-            allowlist = false
-          [probe]
-            exportname = /tmp/nbd-export.img
-          EOF
-          sudo pkill nbd-server || true
-          sudo nbd-server -C /tmp/nbd.conf
-          sleep 1
-
-          echo "--- devices before ---"; N_BEFORE=$(ls /dev/nbd* | wc -l)
-
-          # Ask for an index that does not exist yet; netlink-capable
-          # nbd-client makes the kernel create it on demand.
-          sudo nbd-client -N probe 127.0.0.1 /dev/nbd200
-          ls -la /dev/nbd200
-          sudo nbd-client -d /dev/nbd200
-
-          echo "--- bulk creation timing (50 devices) ---"
-          T0=$(date +%s)
-          for i in $(seq 300 349); do
-            sudo nbd-client -N probe 127.0.0.1 "/dev/nbd$i" >/dev/null 2>&1
-            sudo nbd-client -d "/dev/nbd$i" >/dev/null 2>&1
-          done
-          T1=$(date +%s)
-          echo "50 devices in $((T1-T0))s"
-          N_AFTER=$(ls /dev/nbd* | wc -l)
-          echo "device count: $N_BEFORE -> $N_AFTER"
-          [ -b /dev/nbd349 ]
-
-      - name: "arm64: build nbd.ko from source and insmod"
-        if: matrix.arch == 'arm64'
-        run: |
-          set -euxo pipefail
-          T0=$(date +%s)
-          KVER="$(uname -r)"
-          BASE="${KVER%%-*}"
-          sudo apt-get update -qq
-          sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev
-          cd /tmp
-          curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz"
-          tar -xf "linux-${BASE}.tar.xz"
-          cd "linux-${BASE}"
-          zcat /proc/config.gz > .config
-          ./scripts/config -m BLK_DEV_NBD
-          ./scripts/config --disable MODULE_SIG_ALL || true
-          ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true
-          make olddefconfig
-          grep -E '(MODVERSIONS|MODULE_SIG|BLK_DEV_NBD|LOCALVERSION)[ =]' .config || true
-          make -j"$(nproc)" modules_prepare
-          # No Module.symvers for the running kernel: modpost cannot resolve
-          # core symbols and errors out; WARN=1 downgrades that. Runtime
-          # symbol resolution happens in the kernel at insmod.
-          make -j"$(nproc)" KBUILD_MODPOST_WARN=1 M=drivers/block modules
-          modinfo -F vermagic drivers/block/nbd.ko
-          echo "running kernel: $KVER"
-          if ! sudo insmod drivers/block/nbd.ko nbds_max=256; then
-            echo "insmod failed, dmesg:"
-            sudo dmesg | tail -15
-            exit 1
-          fi
-          T1=$(date +%s)
-          echo "TOTAL arm nbd provisioning: $((T1-T0))s"
-          ls /dev/nbd* | wc -l
-          [ -b /dev/nbd0 ] && [ -b /dev/nbd255 ]
-          # size the cacheable artifact
-          ls -la drivers/block/nbd.ko
-
-      - name: Verdict
-        run: |
-          set -x
-          echo "=== VERDICT for ${{ matrix.runner }} ==="
-          ls /dev/nbd* 2>/dev/null | wc -l
-          [ -e /dev/kvm ] && echo KVM=yes || echo KVM=no
-          sudo dmesg | grep -iE 'nbd|taint' | tail -5 || true
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
index a84d82118a..75449a648e 100644
--- a/.github/workflows/integration_tests.yml
+++ b/.github/workflows/integration_tests.yml
@@ -16,14 +16,14 @@ on:
       CODECOV_TOKEN: { required: false }
 jobs:
   run:
-    # Blacksmith x64 kernels ship nbd built in (16 devices; init-client.sh
-    # skips its modprobe in that case) and expose /dev/kvm for Firecracker.
-    # The orchestrator's NBD pool clamps to the available device count, so
-    # 16 caps concurrent sandboxes - watch for NoFreeSlotsError ("no free
-    # slots") in service logs if the suite grows more parallel.
-    # 32vcpu matches the 32-core infra-tests runner this previously ran on.
+    # Stays on infra-tests: Blacksmith's 6.6 kernel lacks
+    # UFFD_FEATURE_WP_ASYNC (Linux >= 6.7), which sandbox resume
+    # hard-requires - snapshot loads fail with "Failed to UFFD object".
+    # Everything else is provisioned for (nbd is built into their x64
+    # kernel; init-client.sh tolerates that and pre-existing swap).
+    # Revisit when Blacksmith ships a >= 6.7 kernel.
     if: ${{ inputs.run-tests == true }}
-    runs-on: blacksmith-32vcpu-ubuntu-2404
+    runs-on: infra-tests
     timeout-minutes: 30
     permissions:
       contents: read
diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml
index 73273abbeb..da64bb6802 100644
--- a/.github/workflows/pr-tests-arm64.yml
+++ b/.github/workflows/pr-tests-arm64.yml
@@ -73,11 +73,12 @@ jobs:
   arm64-unit-tests:
     name: Run ARM64 test shards
     if: ${{ inputs.run-tests == true }}
-    # All shards run on Blacksmith arm64. The orchestrator shard needs the
-    # nbd driver, which the Blacksmith arm64 kernel does not ship - the
-    # ensure-nbd action builds it from source (cached per kernel release).
-    # There is no /dev/kvm on any arm64 runner (GitHub's included), so the
-    # KVM smoketest skips there either way.
+    # Shards that only need Docker run on Blacksmith arm64. The orchestrator
+    # shard stays on infra-runner-arm: Blacksmith's 6.6 kernel lacks
+    # UFFD_FEATURE_WP_ASYNC (Linux >= 6.7), which the uffd memory backend
+    # hard-requires. nbd is solvable there (the ensure-nbd action can build
+    # it from source); the kernel version is not. Revisit when Blacksmith
+    # ships a >= 6.7 kernel.
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 45
     env:
@@ -123,7 +124,7 @@ jobs:
             flag: arm64-orchestrator
             test_path: ./...
             sudo: true
-            runner: blacksmith-4vcpu-ubuntu-2404-arm
+            runner: infra-runner-arm
           - package: packages/shared
             flag: arm64-shared
             test_path: ./pkg/...
diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml
index 61171eef60..8eeb0162fe 100644
--- a/.github/workflows/pr-tests.yml
+++ b/.github/workflows/pr-tests.yml
@@ -18,10 +18,11 @@ jobs:
   run-tests:
     name: Run unit test shards
     if: ${{ inputs.run-tests == true }}
-    # All shards run on Blacksmith. The orchestrator shard needs the nbd
-    # driver (built into the Blacksmith x64 kernel; see ensure-nbd action)
-    # and /dev/kvm (present on Blacksmith x64 - verify the smoketest RUNS,
-    # it skips silently when /dev/kvm is missing).
+    # Shards that only need Docker run on Blacksmith. The orchestrator shard
+    # stays on infra-tests: Blacksmith's 6.6 kernel lacks UFFD_FEATURE_WP_ASYNC
+    # (Linux >= 6.7), which the uffd memory backend and our Firecracker
+    # dirty-page tracking hard-require (nbd/KVM are NOT the blocker - see the
+    # ensure-nbd action). Revisit when Blacksmith ships a >= 6.7 kernel.
     runs-on: ${{ matrix.runner }}
     env:
       GIN_MODE: test
@@ -62,13 +63,11 @@ jobs:
             test_path: ./...
             sudo: true
             runner: blacksmith-4vcpu-ubuntu-2404
-          # 8vcpu: this shard boots Firecracker VMs (smoketest); it ran on a
-          # 32-core infra-tests runner before - bump if the 20m timeout nears.
           - package: packages/orchestrator
             flag: unit-orchestrator
             test_path: ./...
             sudo: true
-            runner: blacksmith-8vcpu-ubuntu-2404
+            runner: infra-tests
           - package: packages/shared
             flag: unit-shared
             test_path: ./pkg/...