diff --git a/.github/actions/ensure-nbd/action.yml b/.github/actions/ensure-nbd/action.yml new file mode 100644 index 0000000000..73271cfdca --- /dev/null +++ b/.github/actions/ensure-nbd/action.yml @@ -0,0 +1,112 @@ +name: "Ensure NBD devices" +description: | + Makes the nbd block driver available with device nodes for the + orchestrator's DevicePool (which sizes itself from + /sys/module/nbd/parameters/nbds_max and refuses to start without it). + Handles, in order: + 1. driver already present (loaded module OR built into the kernel, + e.g. Blacksmith x64 runners) - nothing to do, the existing device + count applies; + 2. stock kernel with the module shipped - plain modprobe; + 3. stock Ubuntu kernel with nbd in linux-modules-extra (GitHub-hosted + arm64 runners) - apt install, then modprobe; + 4. custom kernel without an nbd module at all (Blacksmith arm64 + runners) - build nbd.ko from the matching kernel.org source using + the running kernel's /proc/config.gz, insmod it. The built .ko is + cached per kernel release (~36s to build cold, instant when cached). + +inputs: + nbds-max: + description: "Device count to request when loading the module (no effect when the driver is built in)" + required: false + default: "4096" + +runs: + using: "composite" + steps: + - name: Detect nbd provisioning strategy + id: detect + shell: bash + run: | + set -euo pipefail + if [ -r /sys/module/nbd/parameters/nbds_max ]; then + echo "nbd driver already present (nbds_max=$(cat /sys/module/nbd/parameters/nbds_max))" + echo "strategy=none" >> "$GITHUB_OUTPUT" + elif sudo modprobe nbd nbds_max='${{ inputs.nbds-max }}' 2>/dev/null; then + echo "strategy=modprobe" >> "$GITHUB_OUTPUT" + elif apt-get -s install "linux-modules-extra-$(uname -r)" >/dev/null 2>&1; then + echo "strategy=apt" >> "$GITHUB_OUTPUT" + else + echo "strategy=build" >> "$GITHUB_OUTPUT" + fi + echo "kver=$(uname -r)" >> "$GITHUB_OUTPUT" + + - name: Install nbd module from linux-modules-extra + if: steps.detect.outputs.strategy == 'apt' + shell: bash + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y "linux-modules-extra-$(uname -r)" + sudo modprobe nbd nbds_max='${{ inputs.nbds-max }}' + + - name: Restore cached nbd.ko + if: steps.detect.outputs.strategy == 'build' + uses: actions/cache@v5 + with: + path: /tmp/nbd-ko + key: nbd-ko-${{ runner.arch }}-${{ steps.detect.outputs.kver }} + + - name: Build nbd.ko from kernel source and insmod + if: steps.detect.outputs.strategy == 'build' + shell: bash + run: | + set -euo pipefail + KVER="$(uname -r)" + if [ ! -f /tmp/nbd-ko/nbd.ko ]; then + if [ ! -e /proc/config.gz ]; then + echo "::error::kernel $KVER has no nbd module and no /proc/config.gz to build one from" + exit 1 + fi + sudo apt-get update -qq + sudo apt-get install -y -qq build-essential flex bison bc libssl-dev libelf-dev + BASE="${KVER%%-*}" + BUILD_DIR="$(mktemp -d)" + cd "$BUILD_DIR" + curl -fsSLO "https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${BASE}.tar.xz" + tar -xf "linux-${BASE}.tar.xz" + cd "linux-${BASE}" + zcat /proc/config.gz > .config + ./scripts/config -m BLK_DEV_NBD + # We cannot sign with the kernel's key; the running kernels this + # path targets have CONFIG_MODULE_SIG disabled. + ./scripts/config --disable MODULE_SIG_ALL || true + ./scripts/config --set-str SYSTEM_TRUSTED_KEYS "" || true + make olddefconfig + make -j"$(nproc)" modules_prepare + # Without the running kernel's Module.symvers, modpost cannot + # resolve core symbols and errors out; WARN=1 downgrades that. + # Actual symbol resolution happens in the kernel at insmod. + make -j"$(nproc)" KBUILD_MODPOST_WARN=1 M=drivers/block modules + mkdir -p /tmp/nbd-ko + cp drivers/block/nbd.ko /tmp/nbd-ko/ + fi + sudo insmod /tmp/nbd-ko/nbd.ko nbds_max='${{ inputs.nbds-max }}' || { + echo "insmod failed, dmesg:" + sudo dmesg | tail -15 + exit 1 + } + + - name: Verify devices and quiet udev + shell: bash + run: | + set -euo pipefail + if [ ! -r /sys/module/nbd/parameters/nbds_max ]; then + echo "::error::nbd driver is not available after provisioning" + exit 1 + fi + echo "effective nbds_max=$(cat /sys/module/nbd/parameters/nbds_max)" + # Disable inotify watching of change events for NBD devices + echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules + sudo udevadm control --reload-rules || true + sudo udevadm trigger || true diff --git a/.github/actions/host-init/init-client.sh b/.github/actions/host-init/init-client.sh index 57c4946154..cf5da687c7 100644 --- a/.github/actions/host-init/init-client.sh +++ b/.github/actions/host-init/init-client.sh @@ -17,15 +17,22 @@ sudo mkdir -p /orchestrator/sandbox sudo mkdir -p /orchestrator/template sudo mkdir -p /orchestrator/build -# Add swapfile +# Add swapfile. Some runner images (e.g. Blacksmith) already boot with an +# active /swapfile; fallocate on an in-use swap file fails with ETXTBSY, +# so only create one when no swap is active yet. SWAPFILE="/swapfile" -sudo fallocate -l 1G $SWAPFILE -sudo chmod 600 $SWAPFILE -sudo mkswap $SWAPFILE -sudo swapon $SWAPFILE - -# Make swapfile persistent -echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab +if grep -q '^\S' /proc/swaps 2>/dev/null && [ "$(wc -l < /proc/swaps)" -gt 1 ]; then + echo "swap already active, skipping swapfile creation:" + cat /proc/swaps +else + sudo fallocate -l 1G $SWAPFILE + sudo chmod 600 $SWAPFILE + sudo mkswap $SWAPFILE + sudo swapon $SWAPFILE + + # Make swapfile persistent + echo "$SWAPFILE none swap sw 0 0" | sudo tee -a /etc/fstab +fi # Set swap settings sudo sysctl vm.swappiness=10 @@ -65,8 +72,15 @@ EOH sudo udevadm control --reload-rules sudo udevadm trigger -# Load the nbd module with 4096 devices -sudo modprobe nbd nbds_max=4096 +# Load the nbd module with 4096 devices. Kernels that build nbd in +# (e.g. Blacksmith runners) have no module to load - the parameter file +# already exists and the built-in device count applies; the orchestrator's +# device pool clamps to it. +if [ -r /sys/module/nbd/parameters/nbds_max ]; then + echo "nbd driver already present (nbds_max=$(cat /sys/module/nbd/parameters/nbds_max))" +else + sudo modprobe nbd nbds_max=4096 +fi # Create the directory for the fc mounts mkdir -p /fc-vm diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6d00df773b..75449a648e 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -16,6 +16,12 @@ on: CODECOV_TOKEN: { required: false } jobs: run: + # Stays on infra-tests: Blacksmith's 6.6 kernel lacks + # UFFD_FEATURE_WP_ASYNC (Linux >= 6.7), which sandbox resume + # hard-requires - snapshot loads fail with "Failed to UFFD object". + # Everything else is provisioned for (nbd is built into their x64 + # kernel; init-client.sh tolerates that and pre-existing swap). + # Revisit when Blacksmith ships a >= 6.7 kernel. if: ${{ inputs.run-tests == true }} runs-on: infra-tests timeout-minutes: 30 diff --git a/.github/workflows/pr-tests-arm64.yml b/.github/workflows/pr-tests-arm64.yml index c5ebf44824..da64bb6802 100644 --- a/.github/workflows/pr-tests-arm64.yml +++ b/.github/workflows/pr-tests-arm64.yml @@ -74,7 +74,11 @@ jobs: name: Run ARM64 test shards if: ${{ inputs.run-tests == true }} # Shards that only need Docker run on Blacksmith arm64. The orchestrator - # shard stays on infra-runner-arm: it needs modprobe nbd. + # shard stays on infra-runner-arm: Blacksmith's 6.6 kernel lacks + # UFFD_FEATURE_WP_ASYNC (Linux >= 6.7), which the uffd memory backend + # hard-requires. nbd is solvable there (the ensure-nbd action can build + # it from source); the kernel version is not. Revisit when Blacksmith + # ships a >= 6.7 kernel. runs-on: ${{ matrix.runner }} timeout-minutes: 45 env: @@ -176,16 +180,12 @@ jobs: sudo mkdir -p /mnt/hugepages sudo mount -t hugetlbfs none /mnt/hugepages echo 256 | sudo tee /proc/sys/vm/nr_hugepages + if: matrix.package == 'packages/orchestrator' - # Install extra kernel modules (nbd is not in base modules on GitHub-hosted runners) - sudo apt-get update - sudo apt-get install -y linux-modules-extra-$(uname -r) - sudo modprobe nbd nbds_max=256 - - # Disable inotify watching of change events for NBD devices - echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules - sudo udevadm control --reload-rules - sudo udevadm trigger + - name: Ensure NBD devices + uses: ./.github/actions/ensure-nbd + with: + nbds-max: "256" if: matrix.package == 'packages/orchestrator' - name: Install gotestsum diff --git a/.github/workflows/pr-tests.yml b/.github/workflows/pr-tests.yml index 395e7a44c7..8eeb0162fe 100644 --- a/.github/workflows/pr-tests.yml +++ b/.github/workflows/pr-tests.yml @@ -19,7 +19,10 @@ jobs: name: Run unit test shards if: ${{ inputs.run-tests == true }} # Shards that only need Docker run on Blacksmith. The orchestrator shard - # stays on infra-tests: it needs modprobe nbd and /dev/kvm (smoketest). + # stays on infra-tests: Blacksmith's 6.6 kernel lacks UFFD_FEATURE_WP_ASYNC + # (Linux >= 6.7), which the uffd memory backend and our Firecracker + # dirty-page tracking hard-require (nbd/KVM are NOT the blocker - see the + # ensure-nbd action). Revisit when Blacksmith ships a >= 6.7 kernel. runs-on: ${{ matrix.runner }} env: GIN_MODE: test @@ -120,15 +123,12 @@ jobs: sudo mkdir -p /mnt/hugepages sudo mount -t hugetlbfs none /mnt/hugepages echo 2000 | sudo tee /proc/sys/vm/nr_hugepages + if: matrix.package == 'packages/orchestrator' - # Enable NBD - sudo modprobe nbd nbds_max=256 - - # Disable inotify watching of change events for NBD devices - echo 'ACTION=="add|change", KERNEL=="nbd*", OPTIONS:="nowatch"' | sudo tee /etc/udev/rules.d/97-nbd-device.rules - sudo udevadm control --reload-rules - sudo udevadm trigger - + - name: Ensure NBD devices + uses: ./.github/actions/ensure-nbd + with: + nbds-max: "256" if: matrix.package == 'packages/orchestrator' - name: Install gotestsum diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e6bbbd2039..b1de6c4789 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -44,6 +44,7 @@ jobs: - '.github/workflows/pr-tests.yml' - '.github/workflows/pr-tests-arm64.yml' - '.github/actions/go-setup-cache/**' + - '.github/actions/ensure-nbd/**' # Like test-inputs, but scoped to only the .github paths that affect # the integration harness (its workflow + composite actions) instead # of all of .github/**. This keeps unrelated CI edits (e.g. other