From d67d52fc1fc4f76b456c7412aed68adcfff90dee Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 12:12:50 -0500 Subject: [PATCH 01/17] Add manual ARM64 and Windows platform smoke tests --- .../workflows/kumo-engine-platform-smoke.yml | 224 ++++++++++++++++++ packages/kumo-engine/platform-smoke.md | 90 +++++++ packages/kumo-engine/test/smoke_platform.py | 114 +++++++++ 3 files changed, 428 insertions(+) create mode 100644 .github/workflows/kumo-engine-platform-smoke.yml create mode 100644 packages/kumo-engine/platform-smoke.md create mode 100644 packages/kumo-engine/test/smoke_platform.py diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml new file mode 100644 index 000000000..064ffefe6 --- /dev/null +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Kumo Engine - Platform smoke + +on: # yamllint disable-line rule:truthy + workflow_dispatch: + +permissions: + contents: read + +jobs: + smoke: + name: Engine GPU smoke (${{ matrix.platform }}) + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + include: + - platform: linux-arm64 + group: nv-gpu-arm64-l4-1gpu + label: linux-arm64-gpu-l4-latest-1 + architecture: aarch64 + - platform: windows-wsl + group: nv-gpu-general + label: wsl-amd64-gpu-l4-latest-1 + architecture: x86_64 + runs-on: + group: ${{ matrix.group }} + labels: ${{ matrix.label }} + defaults: + run: + shell: bash + env: + SMOKE_PLATFORM: ${{ matrix.platform }} + EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} + PYTHONUNBUFFERED: '1' + UV_LINK_MODE: copy + KUMO_SMOKE_DEVICE: cuda + steps: + - name: Check runner operating system and GPU + run: | + set -euo pipefail + if [ "$(uname -s)" != Linux ]; then + echo 'This workflow requires a Linux Actions agent, inside WSL for the WSL runner.' >&2 + exit 1 + fi + test "$(uname -m)" = "$EXPECTED_ARCHITECTURE" + if [ "$SMOKE_PLATFORM" = windows-wsl ]; then + if ! grep -qi microsoft /proc/sys/kernel/osrelease; then + echo 'The WSL smoke must execute inside a WSL Linux guest.' >&2 + exit 1 + fi + fi + nvidia-smi + command -v cc + command -v c++ + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Check ARM Faiss build prerequisites + if: matrix.platform == 'linux-arm64' + run: | + set -euo pipefail + cuda_root="${CUDAToolkit_ROOT:-/usr/local/cuda}" + if ! "$cuda_root/bin/nvcc" --version | grep -q 'release 13\.'; then + echo 'ARM GPU Faiss requires CUDA 13 toolkit; set CUDAToolkit_ROOT on the runner.' >&2 + exit 1 + fi + printf '#include \nint main(void) { return 0; }\n' \ + | cc -x c - -lopenblas -o "$RUNNER_TEMP/openblas-check" + echo "CUDAToolkit_ROOT=$cuda_root" >> "$GITHUB_ENV" + + - name: Set up isolated Python 3.12 + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: '3.12' + activate-environment: true + + - name: Set up Rust for DiskGraph + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 + with: + toolchain: stable + + - name: Install and verify CUDA PyTorch + # Both CPython 3.12 architectures are recorded in the repository uv.lock. + run: | + uv pip install 'torch==2.12.0+cu130' --index-url https://download.pytorch.org/whl/cu130 + python - <<'PY' + import torch + + assert torch.cuda.is_available(), 'CUDA must work before building the engine' + print(torch.__version__, torch.cuda.get_device_name(0)) + assert (torch.ones(2, device='cuda') + 1).tolist() == [2, 2] + PY + + - name: Build ARM GPU Faiss + id: faiss + if: matrix.platform == 'linux-arm64' + env: + FAISS_GPU_CACHE_DIR: ${{ runner.temp }}/faiss-arm64 + run: | + set -euo pipefail + wheel_dir="$(scripts/build_faiss_gpu_arm64_wheel.sh --python "$(command -v python)" --cuda cu130)" + echo "wheel-directory=$wheel_dir" >> "$GITHUB_OUTPUT" + + - name: Install the engine and native graph from this checkout + env: + FAISS_WHEEL_DIRECTORY: ${{ steps.faiss.outputs.wheel-directory }} + run: | + set -euo pipefail + faiss_wheels=() + if [ -n "$FAISS_WHEEL_DIRECTORY" ]; then + faiss_wheels=("$FAISS_WHEEL_DIRECTORY"/faiss_gpu-1.14.3-*.whl) + fi + uv pip install --no-sources --torch-backend=cu130 \ + 'torch==2.12.0+cu130' \ + "${faiss_wheels[@]}" \ + ./packages/diskgraph ./packages/kumo-api ./packages/kumo-pql \ + ./packages/kumo-core './packages/kumo-engine[test]' + uv pip check + + - name: Run real ingest, GPU training, and prediction + run: | + python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py \ + --junitxml=platform-gpu-results.xml + + - name: Save smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: engine-gpu-${{ matrix.platform }}-${{ github.run_attempt }} + path: platform-gpu-results.xml + if-no-files-found: ignore + retention-days: 7 + + windows-native: + name: Engine CPU smoke (windows-native) + timeout-minutes: 90 + runs-on: + group: nv-gpu-general + labels: windows-amd64-gpu-l4-latest-1 + defaults: + run: + shell: pwsh + env: + PYTHONUNBUFFERED: '1' + UV_LINK_MODE: copy + KUMO_SMOKE_DEVICE: cpu + steps: + - name: Check native Windows + run: | + if (-not $IsWindows -or $env:PROCESSOR_ARCHITECTURE -ne 'AMD64') { + throw 'This smoke requires a native Windows x64 Actions agent.' + } + + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Activate installed MSVC build tools + run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + if (-not (Test-Path $vswhere)) { + throw 'Runner image needs Visual Studio Build Tools with the x64 C++ toolchain and Windows SDK.' + } + $installation = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath + if ($LASTEXITCODE -ne 0 -or -not $installation) { + throw 'No installed Visual Studio x64 C++ build tools found.' + } + & "$installation\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64 -HostArch amd64 -SkipAutomaticLocation + foreach ($name in @('cl', 'link', 'cmake')) { + Get-Command $name -ErrorAction Stop | Select-Object Name, Source + } + # Keep the activated toolchain available in later Actions steps. + foreach ($name in @('PATH', 'INCLUDE', 'LIB', 'LIBPATH')) { + $value = [Environment]::GetEnvironmentVariable($name) + "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + + - name: Set up isolated Python 3.12 + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + python-version: '3.12' + activate-environment: true + + - name: Set up Rust for DiskGraph + uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 + with: + toolchain: stable-x86_64-pc-windows-msvc + + - name: Install and verify CPU PyTorch + # The CPython 3.12 win_amd64 wheel is recorded in uv.lock. + run: | + uv pip install 'torch==2.12.0+cpu' --index-url https://download.pytorch.org/whl/cpu + if ($LASTEXITCODE -ne 0) { throw 'CPU PyTorch installation failed.' } + python -c "import sys, torch; assert sys.platform == 'win32'; assert torch.version.cuda is None; print(torch.__version__); assert (torch.ones(2, device='cpu') + 1).tolist() == [2, 2]" + if ($LASTEXITCODE -ne 0) { throw 'Native CPU execution failed.' } + + - name: Install the engine and native graph from this checkout + run: | + uv pip install --no-sources --torch-backend=cpu ` + 'torch==2.12.0+cpu' ` + ./packages/diskgraph ./packages/kumo-api ./packages/kumo-pql ` + ./packages/kumo-core './packages/kumo-engine[test]' + if ($LASTEXITCODE -ne 0) { throw 'Engine source installation failed.' } + uv pip check + if ($LASTEXITCODE -ne 0) { throw 'Engine dependency check failed.' } + + - name: Run real ingest, CPU training, and prediction + run: | + python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py --junitxml=platform-cpu-results.xml + + - name: Save smoke results + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: engine-cpu-windows-native-${{ github.run_attempt }} + path: platform-cpu-results.xml + if-no-files-found: ignore + retention-days: 7 diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md new file mode 100644 index 000000000..97385c7d5 --- /dev/null +++ b/packages/kumo-engine/platform-smoke.md @@ -0,0 +1,90 @@ +# Platform smoke + +The manual [Platform smoke workflow](../../.github/workflows/kumo-engine-platform-smoke.yml) +installs the checked-out engine and DiskGraph source, then runs +[smoke_platform.py](test/smoke_platform.py): local Parquet ingestion, a short +training run, and prediction. Linux ARM64 and WSL train on CUDA; native +Windows uses CPU Faiss and CPU training. Failed installation or a missing +requested device fails the job. +This tests source execution; it does not build or publish a release wheel. + +From the repository root, with engine dependencies installed: + +```bash +python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py +# For CPU training (PowerShell: $env:KUMO_SMOKE_DEVICE = 'cpu'): +KUMO_SMOKE_DEVICE=cpu python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py +``` + +Acceptance requires real snapshot ingestion, finite gradients observed +at an optimizer step, finite training metrics, and finite predictions for all +100 unlabeled entities (with 300 labeled entities used for training). +`KUMO_SMOKE_DEVICE` selects `cpu` or `cuda` (default), and +the optimizer check requires gradients on that exact device. CUDA mode sets +`KUMO_REQUIRE_GPU=1` and fails instead of skipping without CUDA; CPU mode +explicitly selects CPU training. Its filename excludes it from ordinary test +discovery. Platform, architecture, Python, Torch, requested device, and CUDA +availability are printed for diagnosis. + +Local Linux ARM64 CPU validation passed with four optimizer steps and 100 +finite predictions, using Python 3.14, Torch 2.12.0+cpu, and public PyG 2.8.1. +That run used the checkout's Python source and freshly built Linux DiskGraph +debug binaries. +It does not establish a Windows build. CUDA mode also correctly fails when +CUDA is unavailable. Windows, WSL, and GPU execution remain unverified; the +first manual runner executions must validate their installation and runtime. + +In GitHub Actions, select **Kumo Engine - Platform smoke**, choose **Run +workflow**, and select the branch containing this workflow. All three jobs run +independently. No push or pull-request triggers are enabled. Each dispatch +tests its selected revision. The workflow builds on native portability commit +`e4b423c1aeff1a83564e1a26e58de6939cb01003`. + +| Environment | Runner group | Runner label | +| --- | --- | --- | +| Linux ARM64, L4 | `nv-gpu-arm64-l4-1gpu` | `linux-arm64-gpu-l4-latest-1` | +| Windows x86-64 through WSL, L4 | `nv-gpu-general` | `wsl-amd64-gpu-l4-latest-1` | +| Native Windows x86-64, CPU execution on L4 host | `nv-gpu-general` | `windows-amd64-gpu-l4-latest-1` | + +The repository must have access to these runner groups. All images need a +C/C++ compiler and network access to package indexes and Rust dependencies. +Linux ARM64 and WSL also need a working NVIDIA driver. Python 3.12 and Rust +are installed by the workflow. PyTorch 2.12.0 uses CUDA 13 on Linux/WSL and +the CPU build on native Windows, with wheel versions recorded in `uv.lock`. +Other Python dependencies resolve from package metadata; this is not a fully +locked environment. + +ARM additionally needs CUDA 13 development tools at `/usr/local/cuda` (or +`CUDAToolkit_ROOT`) and OpenBLAS development headers/libraries. The workflow +checks these prerequisites and uses the existing ARM Faiss source builder. +It does not install system packages or alter the runner image. + +The WSL job assumes the Actions agent executes inside the Linux guest. It +checks the Linux kernel for WSL and fails if this runner instead executes +Windows host shells. That registration detail and installed image packages +still need confirmation in the first manual run. Git Bash is not a WSL test. + +The native Windows job runs PowerShell and installs the MSVC Rust target. +Its image must already contain Visual Studio 2022 or newer Build Tools with +the x64 C++ toolchain, Windows SDK, and CMake. The workflow locates the installed +tools with `vswhere`, activates their developer environment, and checks that +`cl`, `link`, and `cmake` are available. It fails explicitly when prerequisites +are absent; it does not install system toolchains or reboot the runner. + +Engine metadata selects `kumo-core[faiss-cpu]` on native Windows and +`kumo-core[faiss-gpu]` elsewhere, including WSL. Only one Faiss distribution +is installed because both provide the same Python import namespace. Windows +CPU Faiss and CPU training do not cover native Windows GPU execution. +DiskGraph and Python portability changes are maintained in this repository. +The native build and runner image still need validation before native +execution can be called operational. + +Native Windows coverage is limited to the isolated engine installation in this +workflow. Root `kumo[cu130]`, `kumo[cu132]`, and full-workspace installs remain +unsupported: root GPU extras also request GPU Faiss, conflicting with CPU +Faiss selection. Root extras are unchanged. + +Native versus WSL wheel distribution remains undecided. Enable PR checks only +after manual runs are +reliable; enterprise runner checks should follow the repository's trusted +copy-pr-bot workflow. diff --git a/packages/kumo-engine/test/smoke_platform.py b/packages/kumo-engine/test/smoke_platform.py new file mode 100644 index 000000000..14c4c8957 --- /dev/null +++ b/packages/kumo-engine/test/smoke_platform.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Explicit smoke: pytest -q -s packages/kumo-engine/test/smoke_platform.py. + +Set KUMO_SMOKE_DEVICE=cpu for CPU training; the default is mandatory CUDA. +The filename keeps this integration check out of ordinary test discovery. +""" + +import json +import math +import os +import platform +import sys + +import numpy as np +import pandas as pd +import torch +from kumoapi.model_plan import ModelPlan +from torch.optim.optimizer import register_optimizer_step_post_hook + +import kumo.engine as kumo +from kumo.engine._scratch import get_scratch_dir + + +def test_platform_train_predict(tmp_path, tmp_path_factory, monkeypatch): + device = os.environ.get('KUMO_SMOKE_DEVICE', 'cuda') + assert device in {'cpu', 'cuda'}, 'KUMO_SMOKE_DEVICE must be cpu or cuda' + diagnostics = { + 'requested_device': device, + 'platform': platform.platform(), + 'architecture': platform.machine(), + 'python': sys.version, + 'torch': torch.__version__, + 'cuda_runtime': torch.version.cuda, + 'cuda_available': torch.cuda.is_available(), + } + print(json.dumps(diagnostics, sort_keys=True), flush=True) + if device == 'cuda': + assert torch.cuda.is_available(), 'This smoke test requires a CUDA GPU' + print(f'CUDA device: {torch.cuda.get_device_name(0)}', flush=True) + monkeypatch.setenv('KUMO_REQUIRE_GPU', '1' if device == 'cuda' else '0') + monkeypatch.setenv('KUMO_ACCELERATOR', device) + + previous = get_scratch_dir() + kumo.configure(scratch_dir=tmp_path / 'scratch') + try: + # A single numerical table exercises real ingestion and training + # without external datasets, credentials, or text-model downloads. + rng = np.random.default_rng(0) + row_count = 400 + labeled_count = 300 + targets = rng.random(row_count) + targets[labeled_count:] = np.nan + source_dir = tmp_path_factory.mktemp('smoke-source') + pd.DataFrame({ + 'ID': np.arange(row_count), + 'Y': targets, + 'F1': rng.normal(size=row_count), + }).to_parquet(source_dir / 'entities.parquet') + connector = kumo.FileConnector(root_dir=str(source_dir)) + entities = kumo.Table.from_source_table( + connector['entities.parquet'], primary_key='ID').infer_metadata() + graph = kumo.Graph(tables={'entities': entities}, edges=[]) + snapshot = graph.snapshot() + query = kumo.PredictiveQuery( + graph=graph, query='PREDICT entities.Y FOR EACH entities.ID') + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.disable_explain = True + plan.optimization.max_epochs = 1 + plan.optimization.max_steps_per_epoch = 2 + trainer = kumo.Trainer(model_plan=plan) + optimizer_steps = [] + + def record_optimizer_step(optimizer, _args, _kwargs): + gradients = [parameter.grad for group in optimizer.param_groups + for parameter in group['params'] + if parameter.grad is not None] + assert gradients, 'Optimizer step did not produce gradients' + assert all(gradient.device.type == device for gradient in gradients) + assert all(torch.isfinite(gradient).all().item() + for gradient in gradients) + optimizer_steps.append(1) + + # Observe real optimizer work; availability alone cannot establish + # that training actually ran on the requested device. + hook = register_optimizer_step_post_hook(record_optimizer_step) + try: + fitted = trainer.fit(query.generate_training_table(snapshot), + snapshot=snapshot) + finally: + hook.remove() + assert optimizer_steps, f'No {device} optimizer step was observed' + assert fitted.metrics + assert all(math.isfinite(value) for value in fitted.metrics.values()) + assert any(name.startswith('train_') for name in fitted.metrics) + + predictions = trainer.predict( + query.generate_prediction_table(snapshot), snapshot=snapshot) + frame = predictions.frame + prediction_count = row_count - labeled_count + assert predictions.num_rows == prediction_count + assert len(frame) == prediction_count + assert set(frame['ENTITY']) == set(range(labeled_count, row_count)) + assert np.isfinite(frame['TARGET_PRED'].to_numpy()).all() + if device == 'cuda': + torch.cuda.synchronize() + print(f'Platform smoke completed: {len(optimizer_steps)} ' + f'{device} optimizer steps, ' + f'{len(frame)} finite predictions', + flush=True) + finally: + kumo.configure(scratch_dir=previous) From 7ec9a9b48c326cad5d987f49b70384abd289010a Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 13:01:18 -0500 Subject: [PATCH 02/17] Enable manual PR76 platform qualification via trusted copy --- .github/workflows/kumo-engine-platform-smoke.yml | 4 ++++ packages/kumo-engine/platform-smoke.md | 11 +++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 064ffefe6..4b46ff4fc 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -5,6 +5,10 @@ name: Kumo Engine - Platform smoke on: # yamllint disable-line rule:truthy + # Temporary pre-merge qualification: manually admitted by copy-pr-bot. + # Remove this PR-specific trigger before merging; retain manual dispatch. + push: + branches: [pull-request/76] workflow_dispatch: permissions: diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 97385c7d5..f19aeca7d 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -36,10 +36,17 @@ first manual runner executions must validate their installation and runtime. In GitHub Actions, select **Kumo Engine - Platform smoke**, choose **Run workflow**, and select the branch containing this workflow. All three jobs run -independently. No push or pull-request triggers are enabled. Each dispatch -tests its selected revision. The workflow builds on native portability commit +independently. Each dispatch tests its selected revision. The workflow builds +on native portability commit `e4b423c1aeff1a83564e1a26e58de6939cb01003`. +Before the workflow is registered, PR 76 has a temporary push trigger scoped +only to its trusted `pull-request/76` copy. A maintainer admits the exact PR +head with `/ok to test `; copy-pr-bot's push starts all three +jobs without changing `main`. Remove this temporary trigger before merge so +the delivered workflow remains manual-only. Ordinary fork pushes and +`pull_request` events do not start these jobs. + | Environment | Runner group | Runner label | | --- | --- | --- | | Linux ARM64, L4 | `nv-gpu-arm64-l4-1gpu` | `linux-arm64-gpu-l4-latest-1` | From da6fab25c9c98ac0bb52945145043cdb4b4c3552 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 13:19:50 -0500 Subject: [PATCH 03/17] Run native Windows CPU smoke on a hosted runner --- .github/workflows/kumo-engine-platform-smoke.yml | 4 +--- packages/kumo-engine/platform-smoke.md | 11 +++++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 4b46ff4fc..9f12a75c6 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -144,9 +144,7 @@ jobs: windows-native: name: Engine CPU smoke (windows-native) timeout-minutes: 90 - runs-on: - group: nv-gpu-general - labels: windows-amd64-gpu-l4-latest-1 + runs-on: windows-2022 defaults: run: shell: pwsh diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index f19aeca7d..251f57574 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -51,9 +51,11 @@ the delivered workflow remains manual-only. Ordinary fork pushes and | --- | --- | --- | | Linux ARM64, L4 | `nv-gpu-arm64-l4-1gpu` | `linux-arm64-gpu-l4-latest-1` | | Windows x86-64 through WSL, L4 | `nv-gpu-general` | `wsl-amd64-gpu-l4-latest-1` | -| Native Windows x86-64, CPU execution on L4 host | `nv-gpu-general` | `windows-amd64-gpu-l4-latest-1` | +| Native Windows x86-64, CPU | GitHub-hosted | `windows-2022` | -The repository must have access to these runner groups. All images need a +The repository must have access to the two GPU runner groups. Native Windows +uses a GitHub-hosted CPU runner and needs no enterprise GPU group access. +All images need a C/C++ compiler and network access to package indexes and Rust dependencies. Linux ARM64 and WSL also need a working NVIDIA driver. Python 3.12 and Rust are installed by the workflow. PyTorch 2.12.0 uses CUDA 13 on Linux/WSL and @@ -72,8 +74,9 @@ Windows host shells. That registration detail and installed image packages still need confirmation in the first manual run. Git Bash is not a WSL test. The native Windows job runs PowerShell and installs the MSVC Rust target. -Its image must already contain Visual Studio 2022 or newer Build Tools with -the x64 C++ toolchain, Windows SDK, and CMake. The workflow locates the installed +The [Windows Server 2022 image](https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md) +includes Visual Studio 2022, the x64 C++ toolchain, Windows SDK, and CMake. +The workflow locates the installed tools with `vswhere`, activates their developer environment, and checks that `cl`, `link`, and `cmake` are available. It fails explicitly when prerequisites are absent; it does not install system toolchains or reboot the runner. From 8be0496aab37050da3d1295bb8c18669412e0878 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 13:22:31 -0500 Subject: [PATCH 04/17] Resolve PyTorch dependencies from their appropriate indexes --- .github/workflows/kumo-engine-platform-smoke.yml | 4 ++-- packages/kumo-engine/platform-smoke.md | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 9f12a75c6..ba5f4db9f 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -92,7 +92,7 @@ jobs: - name: Install and verify CUDA PyTorch # Both CPython 3.12 architectures are recorded in the repository uv.lock. run: | - uv pip install 'torch==2.12.0+cu130' --index-url https://download.pytorch.org/whl/cu130 + uv pip install 'torch==2.12.0+cu130' --torch-backend=cu130 python - <<'PY' import torch @@ -197,7 +197,7 @@ jobs: - name: Install and verify CPU PyTorch # The CPython 3.12 win_amd64 wheel is recorded in uv.lock. run: | - uv pip install 'torch==2.12.0+cpu' --index-url https://download.pytorch.org/whl/cpu + uv pip install 'torch==2.12.0+cpu' --torch-backend=cpu if ($LASTEXITCODE -ne 0) { throw 'CPU PyTorch installation failed.' } python -c "import sys, torch; assert sys.platform == 'win32'; assert torch.version.cuda is None; print(torch.__version__); assert (torch.ones(2, device='cpu') + 1).tolist() == [2, 2]" if ($LASTEXITCODE -ne 0) { throw 'Native CPU execution failed.' } diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 251f57574..964a49cc5 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -81,8 +81,9 @@ tools with `vswhere`, activates their developer environment, and checks that `cl`, `link`, and `cmake` are available. It fails explicitly when prerequisites are absent; it does not install system toolchains or reboot the runner. -Engine metadata selects `kumo-core[faiss-cpu]` on native Windows and -`kumo-core[faiss-gpu]` elsewhere, including WSL. Only one Faiss distribution +Engine metadata selects `kumo-core[faiss]`, whose platform markers choose +CPU Faiss on native Windows and GPU Faiss elsewhere, including WSL. +Only one Faiss distribution is installed because both provide the same Python import namespace. Windows CPU Faiss and CPU training do not cover native Windows GPU execution. DiskGraph and Python portability changes are maintained in this repository. From 2f9f415efd1491a91a279979ffd164fa1524a2a0 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 14:06:26 -0500 Subject: [PATCH 05/17] Expose synthetic ingest failures and cache native Rust dependencies --- .../workflows/kumo-engine-platform-smoke.yml | 9 ++++++ packages/kumo-engine/platform-smoke.md | 30 +++++++++++++------ packages/kumo-engine/test/smoke_platform.py | 6 ++++ 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index ba5f4db9f..1fa8a336a 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -194,6 +194,15 @@ jobs: with: toolchain: stable-x86_64-pc-windows-msvc + - name: Cache native Rust dependencies + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: packages/diskgraph -> target + key: windows-2022-python3.12 + cache-on-failure: 'true' + cache-workspace-crates: 'false' + cache-bin: 'false' + - name: Install and verify CPU PyTorch # The CPython 3.12 win_amd64 wheel is recorded in uv.lock. run: | diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 964a49cc5..13b7dce08 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -30,9 +30,17 @@ Local Linux ARM64 CPU validation passed with four optimizer steps and 100 finite predictions, using Python 3.14, Torch 2.12.0+cpu, and public PyG 2.8.1. That run used the checkout's Python source and freshly built Linux DiskGraph debug binaries. -It does not establish a Windows build. CUDA mode also correctly fails when -CUDA is unavailable. Windows, WSL, and GPU execution remain unverified; the -first manual runner executions must validate their installation and runtime. +CUDA mode also correctly fails when CUDA is unavailable. + +On GitHub-hosted Windows 2022, [run 39655272701](https://github.com/NVIDIA/kumo-engine/actions/runs/39655272701) +passed CPU Torch setup, source installation with CPU Faiss (17m08s cold), +and engine import/test collection after the libc guard fix. Real ingestion +then exited with code 1; production diagnostic redaction hid the cause. +The smoke now prints the bounded subprocess output tail for its generated +fixture and rethrows the error, preserving production privacy and failure +behavior. Windows ingest/train/predict, WSL, and GPU execution remain unverified. +Native Rust dependencies are cached even after smoke failure; workspace +crates still rebuild from checkout source. In GitHub Actions, select **Kumo Engine - Platform smoke**, choose **Run workflow**, and select the branch containing this workflow. All three jobs run @@ -40,7 +48,7 @@ independently. Each dispatch tests its selected revision. The workflow builds on native portability commit `e4b423c1aeff1a83564e1a26e58de6939cb01003`. -Before the workflow is registered, PR 76 has a temporary push trigger scoped +For pre-merge qualification, PR 76 has a temporary push trigger scoped only to its trusted `pull-request/76` copy. A maintainer admits the exact PR head with `/ok to test `; copy-pr-bot's push starts all three jobs without changing `main`. Remove this temporary trigger before merge so @@ -53,7 +61,9 @@ the delivered workflow remains manual-only. Ordinary fork pushes and | Windows x86-64 through WSL, L4 | `nv-gpu-general` | `wsl-amd64-gpu-l4-latest-1` | | Native Windows x86-64, CPU | GitHub-hosted | `windows-2022` | -The repository must have access to the two GPU runner groups. Native Windows +Access to the two GPU runner groups is requested in +[runner configuration PR 549](https://github.com/nv-gha-runners/enterprise-runner-configuration/pull/549); +review and application remain pending. Native Windows uses a GitHub-hosted CPU runner and needs no enterprise GPU group access. All images need a C/C++ compiler and network access to package indexes and Rust dependencies. @@ -87,11 +97,13 @@ Only one Faiss distribution is installed because both provide the same Python import namespace. Windows CPU Faiss and CPU training do not cover native Windows GPU execution. DiskGraph and Python portability changes are maintained in this repository. -The native build and runner image still need validation before native -execution can be called operational. +The full native runtime still needs validation before it can be called +operational. -Native Windows coverage is limited to the isolated engine installation in this -workflow. Root `kumo[cu130]`, `kumo[cu132]`, and full-workspace installs remain +Native Windows coverage is limited to this isolated engine installation with +zero DataLoader worker processes, selected by the current Windows and test +configuration. The existing Unix worker-pool paths are not qualified by this +smoke. Root `kumo[cu130]`, `kumo[cu132]`, and full-workspace installs remain unsupported: root GPU extras also request GPU Faiss, conflicting with CPU Faiss selection. Root extras are unchanged. diff --git a/packages/kumo-engine/test/smoke_platform.py b/packages/kumo-engine/test/smoke_platform.py index 14c4c8957..2b3570151 100644 --- a/packages/kumo-engine/test/smoke_platform.py +++ b/packages/kumo-engine/test/smoke_platform.py @@ -16,6 +16,7 @@ import numpy as np import pandas as pd import torch +from kumo_core.diskgraph.runner import DiskgraphIngestError from kumoapi.model_plan import ModelPlan from torch.optim.optimizer import register_optimizer_step_post_hook @@ -110,5 +111,10 @@ def record_optimizer_step(optimizer, _args, _kwargs): f'{device} optimizer steps, ' f'{len(frame)} finite predictions', flush=True) + except DiskgraphIngestError as error: + # This test uses only generated fixture data; expose its subprocess + # diagnostics without changing production error privacy or retrying. + print(error.private_output_tail, file=sys.stderr, flush=True) + raise finally: kumo.configure(scratch_dir=previous) From b88aff67ec76dc119ddad30a19df32f89351b139 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 14:55:23 -0500 Subject: [PATCH 06/17] Use label routing and existing T4 access for WSL smoke --- .../workflows/kumo-engine-platform-smoke.yml | 8 ++---- packages/kumo-engine/platform-smoke.md | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 1fa8a336a..883f72533 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -23,16 +23,12 @@ jobs: matrix: include: - platform: linux-arm64 - group: nv-gpu-arm64-l4-1gpu label: linux-arm64-gpu-l4-latest-1 architecture: aarch64 - platform: windows-wsl - group: nv-gpu-general - label: wsl-amd64-gpu-l4-latest-1 + label: wsl-amd64-gpu-t4-latest-1 architecture: x86_64 - runs-on: - group: ${{ matrix.group }} - labels: ${{ matrix.label }} + runs-on: ${{ matrix.label }} defaults: run: shell: bash diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 13b7dce08..d1d41bcfc 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -32,13 +32,14 @@ That run used the checkout's Python source and freshly built Linux DiskGraph debug binaries. CUDA mode also correctly fails when CUDA is unavailable. -On GitHub-hosted Windows 2022, [run 39655272701](https://github.com/NVIDIA/kumo-engine/actions/runs/39655272701) -passed CPU Torch setup, source installation with CPU Faiss (17m08s cold), -and engine import/test collection after the libc guard fix. Real ingestion -then exited with code 1; production diagnostic redaction hid the cause. -The smoke now prints the bounded subprocess output tail for its generated -fixture and rethrows the error, preserving production privacy and failure -behavior. Windows ingest/train/predict, WSL, and GPU execution remain unverified. +On GitHub-hosted Windows 2022, [run 39662048847](https://github.com/NVIDIA/kumo-engine/actions/runs/39662048847) +completed real ingestion, one CPU training epoch, and checkpoint reload. +Validation-result reading then failed because a local Windows file URI +incorrectly treated the drive letter as a hostname. The URI producer is now +fixed; Windows prediction and the complete smoke still await qualification. +The smoke prints a bounded subprocess output tail for its generated fixture +on failure, preserving production privacy and failure behavior. +WSL and GPU execution remain unverified. Native Rust dependencies are cached even after smoke failure; workspace crates still rebuild from checkout source. @@ -58,12 +59,16 @@ the delivered workflow remains manual-only. Ordinary fork pushes and | Environment | Runner group | Runner label | | --- | --- | --- | | Linux ARM64, L4 | `nv-gpu-arm64-l4-1gpu` | `linux-arm64-gpu-l4-latest-1` | -| Windows x86-64 through WSL, L4 | `nv-gpu-general` | `wsl-amd64-gpu-l4-latest-1` | +| Windows x86-64 through WSL, T4 | `nv-gpu-amd64-t4-1gpu` | `wsl-amd64-gpu-t4-latest-1` | | Native Windows x86-64, CPU | GitHub-hosted | `windows-2022` | -Access to the two GPU runner groups is requested in +The workflow selects runners by label only; the table records catalog group +membership. The current runner-access configuration already grants the T4 +group used by WSL. ARM64 L4 access is requested in [runner configuration PR 549](https://github.com/nv-gha-runners/enterprise-runner-configuration/pull/549); -review and application remain pending. Native Windows +review and application remain pending. +The missing ARM64 grant is a configuration gap; queued jobs alone do not +establish their live permission or provisioning failure cause. Native Windows uses a GitHub-hosted CPU runner and needs no enterprise GPU group access. All images need a C/C++ compiler and network access to package indexes and Rust dependencies. From f16cd1e233b61be6e9560754d6881c6b4b824746 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 15:07:48 -0500 Subject: [PATCH 07/17] Record successful WSL CUDA smoke qualification --- packages/kumo-engine/platform-smoke.md | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index d1d41bcfc..e10573c90 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -32,14 +32,21 @@ That run used the checkout's Python source and freshly built Linux DiskGraph debug binaries. CUDA mode also correctly fails when CUDA is unavailable. -On GitHub-hosted Windows 2022, [run 39662048847](https://github.com/NVIDIA/kumo-engine/actions/runs/39662048847) -completed real ingestion, one CPU training epoch, and checkpoint reload. -Validation-result reading then failed because a local Windows file URI -incorrectly treated the drive letter as a hostname. The URI producer is now -fixed; Windows prediction and the complete smoke still await qualification. +WSL [run 39665870696](https://github.com/NVIDIA/kumo-engine/actions/runs/39665870696) +passed ingestion, four CUDA optimizer steps, and 100 finite predictions +(one test passed in 11.69 seconds; source installation took 5m11s). +It used WSL2 kernel `6.6.114.1-microsoft`, x86-64, Python 3.12.3, +Torch 2.12.0+cu130, and a Tesla T4 with driver 596.36. + +On GitHub-hosted Windows 2022, [run 39663965343](https://github.com/NVIDIA/kumo-engine/actions/runs/39663965343) +completed real ingestion and CPU fitting, including finite gradients and +training metrics. Prediction publication then failed because ownership-path +normalization mixed Windows backslashes with forward-slash containment checks. +That normalization is now fixed; the complete Windows smoke still awaits +qualification. The smoke prints a bounded subprocess output tail for its generated fixture on failure, preserving production privacy and failure behavior. -WSL and GPU execution remain unverified. +Linux ARM64 CUDA execution remains unverified. Native Rust dependencies are cached even after smoke failure; workspace crates still rebuild from checkout source. @@ -83,10 +90,10 @@ ARM additionally needs CUDA 13 development tools at `/usr/local/cuda` (or checks these prerequisites and uses the existing ARM Faiss source builder. It does not install system packages or alter the runner image. -The WSL job assumes the Actions agent executes inside the Linux guest. It +The WSL job requires the Actions agent to execute inside the Linux guest. It checks the Linux kernel for WSL and fails if this runner instead executes -Windows host shells. That registration detail and installed image packages -still need confirmation in the first manual run. Git Bash is not a WSL test. +Windows host shells. The successful WSL run verified this registration and +its installed prerequisites. Git Bash is not a WSL test. The native Windows job runs PowerShell and installs the MSVC Rust target. The [Windows Server 2022 image](https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md) From b5315f24ada221ecc5b87d17bc7a528d0f6fd2ec Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 15:15:36 -0500 Subject: [PATCH 08/17] Record passing native Windows CPU smoke --- packages/kumo-engine/platform-smoke.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index e10573c90..204991c22 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -38,12 +38,11 @@ passed ingestion, four CUDA optimizer steps, and 100 finite predictions It used WSL2 kernel `6.6.114.1-microsoft`, x86-64, Python 3.12.3, Torch 2.12.0+cu130, and a Tesla T4 with driver 596.36. -On GitHub-hosted Windows 2022, [run 39663965343](https://github.com/NVIDIA/kumo-engine/actions/runs/39663965343) -completed real ingestion and CPU fitting, including finite gradients and -training metrics. Prediction publication then failed because ownership-path -normalization mixed Windows backslashes with forward-slash containment checks. -That normalization is now fixed; the complete Windows smoke still awaits -qualification. +Native Windows [run 39667658574](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574) +passed ingestion, four CPU optimizer steps, and 100 finite predictions +(one test passed in 3.86 seconds; source installation took 4m03s). +It used GitHub-hosted Windows Server 2022, AMD64, Python 3.12.10, +and Torch 2.12.0+cpu, exercising CPU Faiss and the checkout's DiskGraph source. The smoke prints a bounded subprocess output tail for its generated fixture on failure, preserving production privacy and failure behavior. Linux ARM64 CUDA execution remains unverified. @@ -109,8 +108,8 @@ Only one Faiss distribution is installed because both provide the same Python import namespace. Windows CPU Faiss and CPU training do not cover native Windows GPU execution. DiskGraph and Python portability changes are maintained in this repository. -The full native runtime still needs validation before it can be called -operational. +The native pass qualifies this smoke's source-execution path, not the full +runtime or release-wheel distribution. Native Windows coverage is limited to this isolated engine installation with zero DataLoader worker processes, selected by the current Windows and test From 716918ffab76fa10333a433b6cae3a14dbe2864c Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 15:29:09 -0500 Subject: [PATCH 09/17] Record repeat WSL pass on qualified Windows revision --- packages/kumo-engine/platform-smoke.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 204991c22..e56c224e1 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -32,9 +32,12 @@ That run used the checkout's Python source and freshly built Linux DiskGraph debug binaries. CUDA mode also correctly fails when CUDA is unavailable. -WSL [run 39665870696](https://github.com/NVIDIA/kumo-engine/actions/runs/39665870696) -passed ingestion, four CUDA optimizer steps, and 100 finite predictions -(one test passed in 11.69 seconds; source installation took 5m11s). +WSL passed twice: [run 39665870696](https://github.com/NVIDIA/kumo-engine/actions/runs/39665870696) +and [run 39667658574](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574). +Both completed ingestion, four CUDA optimizer steps, and 100 finite predictions. +The latest test passed in 25.98 seconds at revision `29b255e`, the same +revision as the native Windows pass below. Observed source installation took +5m11s–11m19s; the latest DiskGraph build accounted for 11m02s. It used WSL2 kernel `6.6.114.1-microsoft`, x86-64, Python 3.12.3, Torch 2.12.0+cu130, and a Tesla T4 with driver 596.36. From 38bf605613f4504cc44b50516f118226c69c2d06 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 21 Oct 2026 16:17:08 -0500 Subject: [PATCH 10/17] Run platform qualification from the upstream development branch --- .../workflows/kumo-engine-platform-smoke.yml | 6 +++--- packages/kumo-engine/platform-smoke.md | 19 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 883f72533..6b280e7ac 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -5,10 +5,10 @@ name: Kumo Engine - Platform smoke on: # yamllint disable-line rule:truthy - # Temporary pre-merge qualification: manually admitted by copy-pr-bot. - # Remove this PR-specific trigger before merging; retain manual dispatch. + # Temporary pre-merge qualification on the upstream development branch. + # Remove this branch-specific trigger before merging; retain manual dispatch. push: - branches: [pull-request/76] + branches: [dm-platform-smoke] workflow_dispatch: permissions: diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index e56c224e1..0f8feb315 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -54,16 +54,15 @@ crates still rebuild from checkout source. In GitHub Actions, select **Kumo Engine - Platform smoke**, choose **Run workflow**, and select the branch containing this workflow. All three jobs run -independently. Each dispatch tests its selected revision. The workflow builds -on native portability commit -`e4b423c1aeff1a83564e1a26e58de6939cb01003`. - -For pre-merge qualification, PR 76 has a temporary push trigger scoped -only to its trusted `pull-request/76` copy. A maintainer admits the exact PR -head with `/ok to test `; copy-pr-bot's push starts all three -jobs without changing `main`. Remove this temporary trigger before merge so -the delivered workflow remains manual-only. Ordinary fork pushes and -`pull_request` events do not start these jobs. +independently. Each dispatch tests its selected revision. The workflow includes +the native portability changes from `dm-platform-native`. + +For pre-merge qualification, a temporary push trigger is scoped to the +upstream `dm-platform-smoke` development branch. Authorized pushes to that +branch start all three jobs without changing `main` or requiring a +copy-pr-bot admission. Remove this temporary trigger before merge so the +delivered workflow remains manual-only. `pull_request` events do not start +these jobs. | Environment | Runner group | Runner label | | --- | --- | --- | From 72dd970dc4a56d5dd1d140cd59e98381275f9272 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Thu, 22 Oct 2026 14:07:48 -0500 Subject: [PATCH 11/17] Exercise Windows path validation and document Faiss platforms --- .../workflows/kumo-engine-platform-smoke.yml | 6 ++++ packages/kumo-engine/platform-smoke.md | 32 +++++++++++++------ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 6b280e7ac..6d696477e 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -217,6 +217,12 @@ jobs: uv pip check if ($LASTEXITCODE -ne 0) { throw 'Engine dependency check failed.' } + - name: Test native Windows path ownership + working-directory: packages/diskgraph + run: | + cargo test --manifest-path Cargo.toml -p diskgraph-ingest --lib disk_manager::path_safety::tests + if ($LASTEXITCODE -ne 0) { throw 'Native Windows path ownership tests failed.' } + - name: Run real ingest, CPU training, and prediction run: | python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py --junitxml=platform-cpu-results.xml diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 0f8feb315..ea5653fdd 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -4,7 +4,7 @@ The manual [Platform smoke workflow](../../.github/workflows/kumo-engine-platfor installs the checked-out engine and DiskGraph source, then runs [smoke_platform.py](test/smoke_platform.py): local Parquet ingestion, a short training run, and prediction. Linux ARM64 and WSL train on CUDA; native -Windows uses CPU Faiss and CPU training. Failed installation or a missing +Windows uses CPU training. Failed installation or a missing requested device fails the job. This tests source execution; it does not build or publish a release wheel. @@ -12,10 +12,20 @@ From the repository root, with engine dependencies installed: ```bash python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py -# For CPU training (PowerShell: $env:KUMO_SMOKE_DEVICE = 'cpu'): +# For CPU training in Bash: KUMO_SMOKE_DEVICE=cpu python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py ``` +For native Windows CPU training in PowerShell: + +```powershell +$env:KUMO_SMOKE_DEVICE = 'cpu' +.venv\Scripts\python.exe -m pytest -q -s packages/kumo-engine/test/smoke_platform.py +``` + +The existing Linux/WSL jobs exercise CUDA; GPU validation is not a requirement +for the planned Windows, WSL, and Linux ARM64 wheel distribution. + Acceptance requires real snapshot ingestion, finite gradients observed at an optimizer step, finite training metrics, and finite predictions for all 100 unlabeled entities (with 300 labeled entities used for training). @@ -104,11 +114,12 @@ tools with `vswhere`, activates their developer environment, and checks that `cl`, `link`, and `cmake` are available. It fails explicitly when prerequisites are absent; it does not install system toolchains or reboot the runner. -Engine metadata selects `kumo-core[faiss]`, whose platform markers choose -CPU Faiss on native Windows and GPU Faiss elsewhere, including WSL. -Only one Faiss distribution -is installed because both provide the same Python import namespace. Windows -CPU Faiss and CPU training do not cover native Windows GPU execution. +Engine metadata requires GPU Faiss on Linux x86-64, including WSL, and CPU +Faiss on macOS. Faiss is optional for native Windows and Linux ARM64; the +existing ARM CUDA job explicitly builds and installs GPU Faiss. The engine +`cpu` extra opts into CPU Faiss on Windows and ARM64. Avoid installing both +Faiss distributions because they share the same Python import namespace. +Native CPU training does not cover native Windows GPU execution. DiskGraph and Python portability changes are maintained in this repository. The native pass qualifies this smoke's source-execution path, not the full runtime or release-wheel distribution. @@ -117,10 +128,11 @@ Native Windows coverage is limited to this isolated engine installation with zero DataLoader worker processes, selected by the current Windows and test configuration. The existing Unix worker-pool paths are not qualified by this smoke. Root `kumo[cu130]`, `kumo[cu132]`, and full-workspace installs remain -unsupported: root GPU extras also request GPU Faiss, conflicting with CPU -Faiss selection. Root extras are unchanged. +outside this smoke's coverage: root GPU extras request GPU Faiss. +Root extras are unchanged. -Native versus WSL wheel distribution remains undecided. Enable PR checks only +Wheel distribution is planned for native Windows, WSL, and Linux ARM64. +Enable PR checks only after manual runs are reliable; enterprise runner checks should follow the repository's trusted copy-pr-bot workflow. From ac75d5762413a1fd129edadf22f58efd651ebb07 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 30 Oct 2026 11:38:37 -0500 Subject: [PATCH 12/17] Qualify installed platform wheels with CPU training MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse exact build artifacts across Python 3.11–3.14 on native Windows, WSL, and Linux ARM64. Test outside the checkout in clean CPU environments, bind installed-package identity to the wheel, and require real ingestion, optimizer work, and finite predictions before release attachment. --- .../workflows/kumo-engine-platform-smoke.yml | 270 +++++------------- .github/workflows/kumo-engine-wheel.yml | 6 +- packages/kumo-engine/platform-smoke.md | 188 ++++-------- .../scripts/verify_installed_wheel.py | 109 +++++++ packages/kumo-engine/test/smoke_platform.py | 19 +- test/scripts/test_engine_wheel_workflow.py | 38 ++- test/scripts/test_installed_wheel.py | 62 ++++ 7 files changed, 337 insertions(+), 355 deletions(-) create mode 100644 packages/kumo-engine/scripts/verify_installed_wheel.py create mode 100644 test/scripts/test_installed_wheel.py diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index 6d696477e..fa0e8759e 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -2,236 +2,104 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -name: Kumo Engine - Platform smoke +name: Kumo Engine - Platform wheel CPU qualification on: # yamllint disable-line rule:truthy - # Temporary pre-merge qualification on the upstream development branch. - # Remove this branch-specific trigger before merging; retain manual dispatch. - push: - branches: [dm-platform-smoke] - workflow_dispatch: + workflow_call: permissions: contents: read jobs: smoke: - name: Engine GPU smoke (${{ matrix.platform }}) - timeout-minutes: 90 + name: CPU wheel (${{ matrix.target.platform }}, Python ${{ matrix.python }}) + timeout-minutes: 60 strategy: fail-fast: false matrix: - include: - - platform: linux-arm64 - label: linux-arm64-gpu-l4-latest-1 - architecture: aarch64 + python: ['3.11', '3.12', '3.13', '3.14'] + target: + - platform: windows-native + runner: windows-2022 + artifact: windows-x86_64 - platform: windows-wsl - label: wsl-amd64-gpu-t4-latest-1 - architecture: x86_64 - runs-on: ${{ matrix.label }} + runner: wsl-amd64-gpu-t4-latest-1 + artifact: x86_64 + - platform: linux-arm64 + runner: ubuntu-24.04-arm + artifact: aarch64 + runs-on: ${{ matrix.target.runner }} defaults: run: shell: bash env: - SMOKE_PLATFORM: ${{ matrix.platform }} - EXPECTED_ARCHITECTURE: ${{ matrix.architecture }} PYTHONUNBUFFERED: '1' + PYTHONPATH: '' + PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1' + CUDA_VISIBLE_DEVICES: '' UV_LINK_MODE: copy - KUMO_SMOKE_DEVICE: cuda + UV_NO_CONFIG: 'true' + SMOKE_PLATFORM: ${{ matrix.target.platform }} + SMOKE_PYTHON: ${{ matrix.python }} + CANDIDATE_SHA: ${{ github.sha }} steps: - - name: Check runner operating system and GPU - run: | - set -euo pipefail - if [ "$(uname -s)" != Linux ]; then - echo 'This workflow requires a Linux Actions agent, inside WSL for the WSL runner.' >&2 - exit 1 - fi - test "$(uname -m)" = "$EXPECTED_ARCHITECTURE" - if [ "$SMOKE_PLATFORM" = windows-wsl ]; then - if ! grep -qi microsoft /proc/sys/kernel/osrelease; then - echo 'The WSL smoke must execute inside a WSL Linux guest.' >&2 - exit 1 - fi - fi - nvidia-smi - command -v cc - command -v c++ - - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Checkout the candidate test harness + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: + ref: ${{ github.sha }} persist-credentials: false - - name: Check ARM Faiss build prerequisites - if: matrix.platform == 'linux-arm64' - run: | - set -euo pipefail - cuda_root="${CUDAToolkit_ROOT:-/usr/local/cuda}" - if ! "$cuda_root/bin/nvcc" --version | grep -q 'release 13\.'; then - echo 'ARM GPU Faiss requires CUDA 13 toolkit; set CUDAToolkit_ROOT on the runner.' >&2 - exit 1 - fi - printf '#include \nint main(void) { return 0; }\n' \ - | cc -x c - -lopenblas -o "$RUNNER_TEMP/openblas-check" - echo "CUDAToolkit_ROOT=$cuda_root" >> "$GITHUB_ENV" - - - name: Set up isolated Python 3.12 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + - name: Download the candidate wheel from this run + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - python-version: '3.12' - activate-environment: true + name: kumo-engine-wheel-${{ matrix.target.artifact }}-${{ github.sha }} + path: ${{ runner.temp }}/platform-wheel/artifacts - - name: Set up Rust for DiskGraph - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 + - name: Set up uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - toolchain: stable + enable-cache: false - - name: Install and verify CUDA PyTorch - # Both CPython 3.12 architectures are recorded in the repository uv.lock. - run: | - uv pip install 'torch==2.12.0+cu130' --torch-backend=cu130 - python - <<'PY' - import torch - - assert torch.cuda.is_available(), 'CUDA must work before building the engine' - print(torch.__version__, torch.cuda.get_device_name(0)) - assert (torch.ones(2, device='cuda') + 1).tolist() == [2, 2] - PY - - - name: Build ARM GPU Faiss - id: faiss - if: matrix.platform == 'linux-arm64' - env: - FAISS_GPU_CACHE_DIR: ${{ runner.temp }}/faiss-arm64 - run: | - set -euo pipefail - wheel_dir="$(scripts/build_faiss_gpu_arm64_wheel.sh --python "$(command -v python)" --cuda cu130)" - echo "wheel-directory=$wheel_dir" >> "$GITHUB_OUTPUT" - - - name: Install the engine and native graph from this checkout - env: - FAISS_WHEEL_DIRECTORY: ${{ steps.faiss.outputs.wheel-directory }} + - name: Clean install and qualify the finished wheel run: | set -euo pipefail - faiss_wheels=() - if [ -n "$FAISS_WHEEL_DIRECTORY" ]; then - faiss_wheels=("$FAISS_WHEEL_DIRECTORY"/faiss_gpu-1.14.3-*.whl) + unset PYTHONPATH + stage="$RUNNER_TEMP/platform-wheel" + if [ "$RUNNER_OS" = Windows ]; then + stage="$(cygpath -u "$stage")" fi - uv pip install --no-sources --torch-backend=cu130 \ - 'torch==2.12.0+cu130' \ - "${faiss_wheels[@]}" \ - ./packages/diskgraph ./packages/kumo-api ./packages/kumo-pql \ - ./packages/kumo-core './packages/kumo-engine[test]' - uv pip check - - - name: Run real ingest, GPU training, and prediction - run: | - python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py \ - --junitxml=platform-gpu-results.xml - - - name: Save smoke results - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: engine-gpu-${{ matrix.platform }}-${{ github.run_attempt }} - path: platform-gpu-results.xml - if-no-files-found: ignore - retention-days: 7 - - windows-native: - name: Engine CPU smoke (windows-native) - timeout-minutes: 90 - runs-on: windows-2022 - defaults: - run: - shell: pwsh - env: - PYTHONUNBUFFERED: '1' - UV_LINK_MODE: copy - KUMO_SMOKE_DEVICE: cpu - steps: - - name: Check native Windows - run: | - if (-not $IsWindows -or $env:PROCESSOR_ARCHITECTURE -ne 'AMD64') { - throw 'This smoke requires a native Windows x64 Actions agent.' - } - - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - with: - persist-credentials: false - - - name: Activate installed MSVC build tools - run: | - $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" - if (-not (Test-Path $vswhere)) { - throw 'Runner image needs Visual Studio Build Tools with the x64 C++ toolchain and Windows SDK.' - } - $installation = & $vswhere -latest -products '*' -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath - if ($LASTEXITCODE -ne 0 -or -not $installation) { - throw 'No installed Visual Studio x64 C++ build tools found.' - } - & "$installation\Common7\Tools\Launch-VsDevShell.ps1" -Arch amd64 -HostArch amd64 -SkipAutomaticLocation - foreach ($name in @('cl', 'link', 'cmake')) { - Get-Command $name -ErrorAction Stop | Select-Object Name, Source - } - # Keep the activated toolchain available in later Actions steps. - foreach ($name in @('PATH', 'INCLUDE', 'LIB', 'LIBPATH')) { - $value = [Environment]::GetEnvironmentVariable($name) - "$name=$value" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - } - - - name: Set up isolated Python 3.12 - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - with: - python-version: '3.12' - activate-environment: true - - - name: Set up Rust for DiskGraph - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 - with: - toolchain: stable-x86_64-pc-windows-msvc - - - name: Cache native Rust dependencies - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 - with: - workspaces: packages/diskgraph -> target - key: windows-2022-python3.12 - cache-on-failure: 'true' - cache-workspace-crates: 'false' - cache-bin: 'false' - - - name: Install and verify CPU PyTorch - # The CPython 3.12 win_amd64 wheel is recorded in uv.lock. - run: | - uv pip install 'torch==2.12.0+cpu' --torch-backend=cpu - if ($LASTEXITCODE -ne 0) { throw 'CPU PyTorch installation failed.' } - python -c "import sys, torch; assert sys.platform == 'win32'; assert torch.version.cuda is None; print(torch.__version__); assert (torch.ones(2, device='cpu') + 1).tolist() == [2, 2]" - if ($LASTEXITCODE -ne 0) { throw 'Native CPU execution failed.' } - - - name: Install the engine and native graph from this checkout - run: | - uv pip install --no-sources --torch-backend=cpu ` - 'torch==2.12.0+cpu' ` - ./packages/diskgraph ./packages/kumo-api ./packages/kumo-pql ` - ./packages/kumo-core './packages/kumo-engine[test]' - if ($LASTEXITCODE -ne 0) { throw 'Engine source installation failed.' } - uv pip check - if ($LASTEXITCODE -ne 0) { throw 'Engine dependency check failed.' } - - - name: Test native Windows path ownership - working-directory: packages/diskgraph - run: | - cargo test --manifest-path Cargo.toml -p diskgraph-ingest --lib disk_manager::path_safety::tests - if ($LASTEXITCODE -ne 0) { throw 'Native Windows path ownership tests failed.' } - - - name: Run real ingest, CPU training, and prediction - run: | - python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py --junitxml=platform-cpu-results.xml - - - name: Save smoke results + mkdir -p "$stage/results" "$stage/test" + cp packages/kumo-engine/test/smoke_platform.py "$stage/test/" + cp packages/kumo-engine/scripts/verify_installed_wheel.py "$stage/test/" + cd "$stage" + wheels=(artifacts/kumo_engine-*.whl) + if [ "${#wheels[@]}" -ne 1 ] || [ ! -f "${wheels[0]}" ]; then + echo 'Expected exactly one candidate engine wheel.' >&2 + exit 1 + fi + uv venv --python "$SMOKE_PYTHON" --seed venv + if [ "$RUNNER_OS" = Windows ]; then + python="$stage/venv/Scripts/python.exe" + else + python="$stage/venv/bin/python" + fi + uv pip install --python "$python" --torch-backend=cpu \ + "${wheels[0]}" pytest 2>&1 | tee results/install.log + uv pip check --python "$python" 2>&1 | tee results/dependencies.log + "$python" test/verify_installed_wheel.py \ + --wheel "${wheels[0]}" --platform "$SMOKE_PLATFORM" \ + --python-version "$SMOKE_PYTHON" --commit "$CANDIDATE_SHA" \ + --output results/environment.json + cd test + "$python" -m pytest -q -s --confcutdir=. \ + --junitxml=../results/junit.xml smoke_platform.py \ + 2>&1 | tee ../results/smoke.log + + - name: Save platform qualification evidence if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: engine-cpu-windows-native-${{ github.run_attempt }} - path: platform-cpu-results.xml - if-no-files-found: ignore - retention-days: 7 + name: engine-wheel-cpu-${{ matrix.target.platform }}-py${{ matrix.python }}-${{ github.sha }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/platform-wheel/results + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/kumo-engine-wheel.yml b/.github/workflows/kumo-engine-wheel.yml index 7e70aeda2..e66a7d0b3 100644 --- a/.github/workflows/kumo-engine-wheel.yml +++ b/.github/workflows/kumo-engine-wheel.yml @@ -239,8 +239,12 @@ jobs: with: wheel_artifact: kumo-engine-wheel-x86_64-${{ github.sha }} + platform-qualification: + needs: [build] + uses: ./.github/workflows/kumo-engine-platform-smoke.yml + attach-to-release: - needs: [build, check-requirements, wheel-smoke] + needs: [build, check-requirements, wheel-smoke, platform-qualification] if: ${{ github.event_name == 'release' }} strategy: fail-fast: false diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index ea5653fdd..2f230d061 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -1,138 +1,50 @@ -# Platform smoke - -The manual [Platform smoke workflow](../../.github/workflows/kumo-engine-platform-smoke.yml) -installs the checked-out engine and DiskGraph source, then runs -[smoke_platform.py](test/smoke_platform.py): local Parquet ingestion, a short -training run, and prediction. Linux ARM64 and WSL train on CUDA; native -Windows uses CPU training. Failed installation or a missing -requested device fails the job. -This tests source execution; it does not build or publish a release wheel. - -From the repository root, with engine dependencies installed: - -```bash -python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py -# For CPU training in Bash: -KUMO_SMOKE_DEVICE=cpu python -m pytest -q -s packages/kumo-engine/test/smoke_platform.py -``` - -For native Windows CPU training in PowerShell: - -```powershell -$env:KUMO_SMOKE_DEVICE = 'cpu' -.venv\Scripts\python.exe -m pytest -q -s packages/kumo-engine/test/smoke_platform.py -``` - -The existing Linux/WSL jobs exercise CUDA; GPU validation is not a requirement -for the planned Windows, WSL, and Linux ARM64 wheel distribution. - -Acceptance requires real snapshot ingestion, finite gradients observed -at an optimizer step, finite training metrics, and finite predictions for all -100 unlabeled entities (with 300 labeled entities used for training). -`KUMO_SMOKE_DEVICE` selects `cpu` or `cuda` (default), and -the optimizer check requires gradients on that exact device. CUDA mode sets -`KUMO_REQUIRE_GPU=1` and fails instead of skipping without CUDA; CPU mode -explicitly selects CPU training. Its filename excludes it from ordinary test -discovery. Platform, architecture, Python, Torch, requested device, and CUDA -availability are printed for diagnosis. - -Local Linux ARM64 CPU validation passed with four optimizer steps and 100 -finite predictions, using Python 3.14, Torch 2.12.0+cpu, and public PyG 2.8.1. -That run used the checkout's Python source and freshly built Linux DiskGraph -debug binaries. -CUDA mode also correctly fails when CUDA is unavailable. - -WSL passed twice: [run 39665870696](https://github.com/NVIDIA/kumo-engine/actions/runs/39665870696) -and [run 39667658574](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574). -Both completed ingestion, four CUDA optimizer steps, and 100 finite predictions. -The latest test passed in 25.98 seconds at revision `29b255e`, the same -revision as the native Windows pass below. Observed source installation took -5m11s–11m19s; the latest DiskGraph build accounted for 11m02s. -It used WSL2 kernel `6.6.114.1-microsoft`, x86-64, Python 3.12.3, -Torch 2.12.0+cu130, and a Tesla T4 with driver 596.36. - -Native Windows [run 39667658574](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574) -passed ingestion, four CPU optimizer steps, and 100 finite predictions -(one test passed in 3.86 seconds; source installation took 4m03s). -It used GitHub-hosted Windows Server 2022, AMD64, Python 3.12.10, -and Torch 2.12.0+cpu, exercising CPU Faiss and the checkout's DiskGraph source. -The smoke prints a bounded subprocess output tail for its generated fixture -on failure, preserving production privacy and failure behavior. -Linux ARM64 CUDA execution remains unverified. -Native Rust dependencies are cached even after smoke failure; workspace -crates still rebuild from checkout source. - -In GitHub Actions, select **Kumo Engine - Platform smoke**, choose **Run -workflow**, and select the branch containing this workflow. All three jobs run -independently. Each dispatch tests its selected revision. The workflow includes -the native portability changes from `dm-platform-native`. - -For pre-merge qualification, a temporary push trigger is scoped to the -upstream `dm-platform-smoke` development branch. Authorized pushes to that -branch start all three jobs without changing `main` or requiring a -copy-pr-bot admission. Remove this temporary trigger before merge so the -delivered workflow remains manual-only. `pull_request` events do not start -these jobs. - -| Environment | Runner group | Runner label | -| --- | --- | --- | -| Linux ARM64, L4 | `nv-gpu-arm64-l4-1gpu` | `linux-arm64-gpu-l4-latest-1` | -| Windows x86-64 through WSL, T4 | `nv-gpu-amd64-t4-1gpu` | `wsl-amd64-gpu-t4-latest-1` | -| Native Windows x86-64, CPU | GitHub-hosted | `windows-2022` | - -The workflow selects runners by label only; the table records catalog group -membership. The current runner-access configuration already grants the T4 -group used by WSL. ARM64 L4 access is requested in -[runner configuration PR 549](https://github.com/nv-gha-runners/enterprise-runner-configuration/pull/549); -review and application remain pending. -The missing ARM64 grant is a configuration gap; queued jobs alone do not -establish their live permission or provisioning failure cause. Native Windows -uses a GitHub-hosted CPU runner and needs no enterprise GPU group access. -All images need a -C/C++ compiler and network access to package indexes and Rust dependencies. -Linux ARM64 and WSL also need a working NVIDIA driver. Python 3.12 and Rust -are installed by the workflow. PyTorch 2.12.0 uses CUDA 13 on Linux/WSL and -the CPU build on native Windows, with wheel versions recorded in `uv.lock`. -Other Python dependencies resolve from package metadata; this is not a fully -locked environment. - -ARM additionally needs CUDA 13 development tools at `/usr/local/cuda` (or -`CUDAToolkit_ROOT`) and OpenBLAS development headers/libraries. The workflow -checks these prerequisites and uses the existing ARM Faiss source builder. -It does not install system packages or alter the runner image. - -The WSL job requires the Actions agent to execute inside the Linux guest. It -checks the Linux kernel for WSL and fails if this runner instead executes -Windows host shells. The successful WSL run verified this registration and -its installed prerequisites. Git Bash is not a WSL test. - -The native Windows job runs PowerShell and installs the MSVC Rust target. -The [Windows Server 2022 image](https://github.com/actions/runner-images/blob/main/images/windows/Windows2022-Readme.md) -includes Visual Studio 2022, the x64 C++ toolchain, Windows SDK, and CMake. -The workflow locates the installed -tools with `vswhere`, activates their developer environment, and checks that -`cl`, `link`, and `cmake` are available. It fails explicitly when prerequisites -are absent; it does not install system toolchains or reboot the runner. - -Engine metadata requires GPU Faiss on Linux x86-64, including WSL, and CPU -Faiss on macOS. Faiss is optional for native Windows and Linux ARM64; the -existing ARM CUDA job explicitly builds and installs GPU Faiss. The engine -`cpu` extra opts into CPU Faiss on Windows and ARM64. Avoid installing both -Faiss distributions because they share the same Python import namespace. -Native CPU training does not cover native Windows GPU execution. -DiskGraph and Python portability changes are maintained in this repository. -The native pass qualifies this smoke's source-execution path, not the full -runtime or release-wheel distribution. - -Native Windows coverage is limited to this isolated engine installation with -zero DataLoader worker processes, selected by the current Windows and test -configuration. The existing Unix worker-pool paths are not qualified by this -smoke. Root `kumo[cu130]`, `kumo[cu132]`, and full-workspace installs remain -outside this smoke's coverage: root GPU extras request GPU Faiss. -Root extras are unchanged. - -Wheel distribution is planned for native Windows, WSL, and Linux ARM64. -Enable PR checks only -after manual runs are -reliable; enterprise runner checks should follow the repository's trusted -copy-pr-bot workflow. +# Installed-wheel platform qualification + +The manual [wheel build workflow](../../.github/workflows/kumo-engine-wheel.yml) +builds the candidate once and passes those artifacts to the reusable +[platform smoke workflow](../../.github/workflows/kumo-engine-platform-smoke.yml). +Select **Kumo Engine - Build and publish wheel**, **Run workflow**, and the +candidate branch. Manual runs upload artifacts; they do not publish a release. + +Qualification covers Python 3.11, 3.12, 3.13 and 3.14 on native Windows x86-64, +Windows through WSL x86-64, and Linux ARM64: twelve independent CPU jobs. +WSL installs the same Linux x86-64 wheel as ordinary Linux; it is not a +separate wheel format. GPU availability is not a qualification requirement. +The producer also retains the macOS ARM64 artifact and its existing checks; +this smoke does not add macOS runtime qualification. + +Each consumer creates a fresh virtual environment, installs the downloaded +wheel and public CPU Torch dependencies, and runs dependency consistency +checks. It copies only the smoke test and standalone environment auditor +outside the checkout. The auditor rejects checkout imports, records origins +for engine/core/API/PQL/DiskGraph, verifies the packaged ingestion executable, +and records the wheel filename, SHA-256, candidate revision, Python, platform, +and Torch version. Native Windows and Linux ARM64 must run without Faiss; +WSL retains the wheel's existing Linux x86-64 Faiss dependency. + +[smoke_platform.py](test/smoke_platform.py) creates a synthetic Parquet table, +performs real snapshot ingestion, and selects CPU through the public +`Trainer(..., accelerator='cpu')` API. Acceptance requires four optimizer +steps with finite CPU gradients, finite training metrics, and all 100 expected +finite predictions. Synchronous DataLoader execution limits Windows coverage +to zero worker processes; Unix worker pools and native Windows GPU execution +are outside this qualification. Subprocess diagnostics are printed only for +this generated fixture, preserving production error privacy. + +The WSL job checks the Linux guest kernel. A Windows host shell or ordinary +Linux x86-64 runner cannot substitute for WSL. Consumer jobs install no Rust +compiler and rebuild no native artifacts: missing executables or libraries +must fail the downloaded-wheel check. Logs, environment evidence and pytest +results distinguish packaging failures from training failures. + +Historical [source smoke runs](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574) +passed native Windows CPU and WSL CUDA ingestion/training/prediction. A local +Linux ARM64 development wheel (`4e910365`) passed clean Python 3.12 CPU +installation, dependency checks, origin auditing, and this smoke with four +optimizer steps and 100 predictions, without Faiss. These +results do not qualify the current candidate across the twelve installed-wheel +jobs; current qualification requires successful runs for that candidate. + +PR checks remain a later decision after manual qualification is reliable. +This change does not require an ARM GPU runner grant or change publication +approval policy. diff --git a/packages/kumo-engine/scripts/verify_installed_wheel.py b/packages/kumo-engine/scripts/verify_installed_wheel.py new file mode 100644 index 000000000..e74af30b3 --- /dev/null +++ b/packages/kumo-engine/scripts/verify_installed_wheel.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Record installed-wheel evidence before executing the CPU smoke. + +This standalone audit uses stdlib filesystem operations so it can reject +checkout imports before loading Kumo's own filesystem abstraction. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.metadata +import importlib.util +import json +import os +import platform +import sys +from pathlib import Path +from urllib.parse import urlsplit +from urllib.request import url2pathname + +MODULES = ('kumo.engine', 'kumo_core', 'kumoapi', 'kumopql', 'diskgraph._c') + + +def check_origin(origin: str | None, prefix: Path) -> str: + if origin is None: + raise ValueError('Required module has no installed file origin') + path = Path(origin).resolve() + if not path.is_relative_to(prefix.resolve()): + raise ValueError(f'Module originates outside the virtual environment: {path}') + return str(path) + + +def check_platform(expected: str, system: str, machine: str, release: str) -> None: + arm = machine.lower() in {'aarch64', 'arm64'} + amd = machine.lower() in {'amd64', 'x86_64'} + wsl = 'microsoft' in release.lower() + valid = { + 'windows-native': system == 'Windows' and amd, + 'windows-wsl': system == 'Linux' and amd and wsl, + 'linux-arm64': system == 'Linux' and arm and not wsl, + } + if not valid[expected]: + raise ValueError(f'Unexpected runtime: {system}/{machine}/{release}') + + +def check_install(source: str | None, wheel: Path) -> None: + if source is None: + raise ValueError('Installed distribution lacks direct-wheel provenance') + recorded = urlsplit(json.loads(source)['url']) + if (recorded.scheme != 'file' or recorded.netloc + or recorded.query or recorded.fragment): + raise ValueError('Installed distribution is not from a local wheel') + installed_path = Path(url2pathname(recorded.path)).resolve() + if os.path.normcase(str(installed_path)) != os.path.normcase(str(wheel.resolve())): + raise ValueError('Installed distribution differs from the downloaded wheel') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--wheel', required=True, type=Path) + parser.add_argument('--platform', required=True, + choices=('windows-native', 'windows-wsl', 'linux-arm64')) + parser.add_argument('--python-version', required=True) + parser.add_argument('--commit', required=True) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args() + if sys.prefix == sys.base_prefix: + raise ValueError('Qualification requires a fresh virtual environment') + if os.environ.get('PYTHONPATH') or os.environ.get('DISKGRAPH_INGEST_BIN'): + raise ValueError('PYTHONPATH and DISKGRAPH_INGEST_BIN must be unset') + actual_python = f'{sys.version_info.major}.{sys.version_info.minor}' + if actual_python != args.python_version: + raise ValueError('Python version differs from the qualification matrix') + check_platform(args.platform, platform.system(), platform.machine(), + platform.release()) + origins = {name: check_origin(importlib.import_module(name).__file__, + Path(sys.prefix)) for name in MODULES} + diskgraph = importlib.import_module('diskgraph') + package = Path(check_origin(diskgraph.__file__, Path(sys.prefix))).parent + ingest = package / 'bin' / ('ingest.exe' if sys.platform == 'win32' else 'ingest') + if not ingest.is_file() or not os.access(ingest, os.X_OK): + raise ValueError('Installed wheel ingestion executable is missing') + faiss_present = importlib.util.find_spec('faiss') is not None + if args.platform != 'windows-wsl' and faiss_present: + raise ValueError('Bare Windows/ARM wheel qualification must omit Faiss') + import torch + with args.wheel.open('rb') as wheel_file: + wheel_sha256 = hashlib.file_digest(wheel_file, 'sha256').hexdigest() + check_install(importlib.metadata.distribution('kumo-engine').read_text( + 'direct_url.json'), args.wheel) + report = { + 'commit': args.commit, 'wheel': args.wheel.name, + 'wheel_sha256': wheel_sha256, + 'platform': args.platform, 'system': platform.platform(), + 'python': sys.version, 'executable': sys.executable, + 'origins': origins, 'ingest': str(ingest), 'faiss_present': faiss_present, + 'torch': torch.__version__, 'torch_cuda_build': torch.version.cuda, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report, sort_keys=True), flush=True) + + +if __name__ == '__main__': + main() diff --git a/packages/kumo-engine/test/smoke_platform.py b/packages/kumo-engine/test/smoke_platform.py index 2b3570151..aa459437e 100644 --- a/packages/kumo-engine/test/smoke_platform.py +++ b/packages/kumo-engine/test/smoke_platform.py @@ -3,13 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 """Explicit smoke: pytest -q -s packages/kumo-engine/test/smoke_platform.py. -Set KUMO_SMOKE_DEVICE=cpu for CPU training; the default is mandatory CUDA. +Run only against an installed wheel outside the source checkout. +CPU training uses the public Trainer accelerator option. The filename keeps this integration check out of ordinary test discovery. """ import json import math -import os import platform import sys @@ -25,8 +25,8 @@ def test_platform_train_predict(tmp_path, tmp_path_factory, monkeypatch): - device = os.environ.get('KUMO_SMOKE_DEVICE', 'cuda') - assert device in {'cpu', 'cuda'}, 'KUMO_SMOKE_DEVICE must be cpu or cuda' + device = 'cpu' + monkeypatch.setenv('SYNC_DATA_LOADER', '1') diagnostics = { 'requested_device': device, 'platform': platform.platform(), @@ -37,11 +37,6 @@ def test_platform_train_predict(tmp_path, tmp_path_factory, monkeypatch): 'cuda_available': torch.cuda.is_available(), } print(json.dumps(diagnostics, sort_keys=True), flush=True) - if device == 'cuda': - assert torch.cuda.is_available(), 'This smoke test requires a CUDA GPU' - print(f'CUDA device: {torch.cuda.get_device_name(0)}', flush=True) - monkeypatch.setenv('KUMO_REQUIRE_GPU', '1' if device == 'cuda' else '0') - monkeypatch.setenv('KUMO_ACCELERATOR', device) previous = get_scratch_dir() kumo.configure(scratch_dir=tmp_path / 'scratch') @@ -71,7 +66,7 @@ def test_platform_train_predict(tmp_path, tmp_path_factory, monkeypatch): plan.training_job.disable_explain = True plan.optimization.max_epochs = 1 plan.optimization.max_steps_per_epoch = 2 - trainer = kumo.Trainer(model_plan=plan) + trainer = kumo.Trainer(model_plan=plan, accelerator='cpu') optimizer_steps = [] def record_optimizer_step(optimizer, _args, _kwargs): @@ -92,7 +87,7 @@ def record_optimizer_step(optimizer, _args, _kwargs): snapshot=snapshot) finally: hook.remove() - assert optimizer_steps, f'No {device} optimizer step was observed' + assert len(optimizer_steps) == 4, 'Expected four CPU optimizer steps' assert fitted.metrics assert all(math.isfinite(value) for value in fitted.metrics.values()) assert any(name.startswith('train_') for name in fitted.metrics) @@ -105,8 +100,6 @@ def record_optimizer_step(optimizer, _args, _kwargs): assert len(frame) == prediction_count assert set(frame['ENTITY']) == set(range(labeled_count, row_count)) assert np.isfinite(frame['TARGET_PRED'].to_numpy()).all() - if device == 'cuda': - torch.cuda.synchronize() print(f'Platform smoke completed: {len(optimizer_steps)} ' f'{device} optimizer steps, ' f'{len(frame)} finite predictions', diff --git a/test/scripts/test_engine_wheel_workflow.py b/test/scripts/test_engine_wheel_workflow.py index 9999da208..f6e486ca1 100644 --- a/test/scripts/test_engine_wheel_workflow.py +++ b/test/scripts/test_engine_wheel_workflow.py @@ -147,8 +147,8 @@ def test_all_platforms_require_validation_and_release_environment(): assert 'pattern' not in download assert download['name'] == _step('build', 'Upload wheel')['with']['name'] assert JOBS['check-requirements']['needs'] == ['build'] - assert set(JOBS['attach-to-release']['needs']) >= { - 'build', 'check-requirements', 'wheel-smoke' + assert set(JOBS['attach-to-release']['needs']) == { + 'build', 'check-requirements', 'wheel-smoke', 'platform-qualification' } assert JOBS['kitmaker']['needs'] == ['attach-to-release'] assert JOBS['kitmaker']['environment'] == 'kumo-engine-release' @@ -180,3 +180,37 @@ def test_publication_forwards_the_shared_prediction_tolerance(): assert call['reload-atol'] == pr_call['reload-atol'] == '0.005' assert run['env']['RELOAD_ATOL'] == '${{ inputs.reload-atol }}' assert '--reload-atol "$RELOAD_ATOL"' in run['run'] + + +def test_cpu_qualification_uses_candidate_artifacts_on_all_pythons(): + caller = JOBS['platform-qualification'] + assert caller['needs'] == ['build'] + assert caller['uses'] == ( + './.github/workflows/kumo-engine-platform-smoke.yml') + path = WORKFLOW.parent / 'kumo-engine-platform-smoke.yml' + workflow = yaml.safe_load(path.read_text()) + # PyYAML's YAML 1.1 parser interprets the GitHub Actions `on` key as True. + assert workflow[True] == {'workflow_call': None} + job = workflow['jobs']['smoke'] + matrix = job['strategy']['matrix'] + assert set(matrix['python']) == {'3.11', '3.12', '3.13', '3.14'} + assert len(matrix['target']) == 3 + assert {row['platform']: row['artifact'] for row in matrix['target']} == { + 'windows-native': 'windows-x86_64', 'windows-wsl': 'x86_64', + 'linux-arm64': 'aarch64', + } + assert {row['artifact'] for row in matrix['target']} <= { + build['arch'] for build in BUILDS} + assert job['env']['CUDA_VISIBLE_DEVICES'] == '' + download = next(step for step in job['steps'] + if step.get('uses', '').startswith( + 'actions/download-artifact@')) + assert download['with'] == { + 'name': ('kumo-engine-wheel-${{ matrix.target.artifact }}-' + '${{ github.sha }}'), + 'path': '${{ runner.temp }}/platform-wheel/artifacts', + } + # Supplying another run or token here could accidentally qualify a stale + # artifact while publishing the current build. + assert 'run-id' not in download['with'] + assert 'github-token' not in download['with'] diff --git a/test/scripts/test_installed_wheel.py b/test/scripts/test_installed_wheel.py new file mode 100644 index 000000000..e2e15b744 --- /dev/null +++ b/test/scripts/test_installed_wheel.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +import runpy +from pathlib import Path + +import pytest + +SCRIPT = (Path(__file__).resolve().parents[2] / + 'packages/kumo-engine/scripts/verify_installed_wheel.py') +HELPER = runpy.run_path(str(SCRIPT)) + + +def test_checkout_origin_rejected(tmp_path): + with pytest.raises(ValueError, match='outside'): + HELPER['check_origin'](str(tmp_path / 'checkout/module.py'), + tmp_path / 'venv') + + +def test_installed_origin_accepted(tmp_path): + origin = tmp_path / 'venv/site-packages/module.py' + assert HELPER['check_origin'](str(origin), tmp_path / 'venv') == str(origin) + + +@pytest.mark.parametrize('expected,system,machine,release', [ + ('windows-native', 'Windows', 'AMD64', '2022'), + ('windows-wsl', 'Linux', 'x86_64', '6.6-microsoft-standard-WSL2'), + ('linux-arm64', 'Linux', 'aarch64', '6.8'), +]) +def test_platform_accepted(expected, system, machine, release): + HELPER['check_platform'](expected, system, machine, release) + + +@pytest.mark.parametrize('system,machine,release', [ + ('Windows', 'AMD64', '2022'), + ('Linux', 'x86_64', '6.8'), + ('Linux', 'aarch64', '6.6-microsoft-standard-WSL2'), +]) +def test_wsl_requires_linux_guest(system, machine, release): + with pytest.raises(ValueError, match='Unexpected runtime'): + HELPER['check_platform']('windows-wsl', system, machine, release) + + +def test_install_matches_artifact(tmp_path): + wheel = tmp_path / 'engine+dev.whl' + source = json.dumps({'url': wheel.as_uri(), 'archive_info': {}}) + HELPER['check_install'](source, wheel) + HELPER['check_install'](source.replace('%2B', '+'), wheel) + with pytest.raises(ValueError, match='differs'): + HELPER['check_install'](source, tmp_path / 'other.whl') + + +def test_install_requires_provenance(tmp_path): + with pytest.raises(ValueError, match='provenance'): + HELPER['check_install'](None, tmp_path / 'engine.whl') + + +def test_install_rejects_remote_artifact(tmp_path): + with pytest.raises(ValueError, match='not from a local'): + HELPER['check_install']('{"url":"https://example.com/engine.whl"}', + tmp_path / 'engine.whl') From ada0791f9b91593ba4d318b26f2d544547324212 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 30 Oct 2026 12:14:39 -0500 Subject: [PATCH 13/17] Document the existing native wheel CPU baseline State the AES and architecture SIMD requirements in installation and release documentation, preserving the existing build configuration. --- docs/development/releasing-kumo-engine.md | 8 ++++++++ packages/kumo-engine/README.md | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index 25fcbab9f..0bdf92e95 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -169,6 +169,14 @@ as a workflow artifact for 30 days. It never attaches or publishes anything: attaching runs only for release events, and PyPI rejects local versions anyway. +## Native CPU baseline + +DiskGraph builds retain the fixed CPU baseline in `.cargo/config.toml` and +the native build backend: AES plus SSE2 on x86-64; AES plus NEON on ARM64. +There is no `target-cpu=native` setting or AVX2 requirement. Qualification +runners and deployment CPUs must expose that baseline, including through +virtualization. + ## Build locally ```bash diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index e425c3bbf..c9ccbecde 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -17,6 +17,10 @@ Kumo Engine (`kumo.engine`) trains graph neural networks on relational data, in- Kumo Engine uses a GPU when one is available: an NVIDIA GPU through CUDA, or an Apple silicon GPU through PyTorch's MPS backend. Without one, training and prediction run on CPU, which is enough for small jobs such as the quickstart below. To choose the device, pass `accelerator=` (`"auto"`, `"cpu"`, `"gpu"`, or `"mps"`) to `Trainer`. +The bundled native engine requires CPU AES instructions: AES and SSE2 on +x86-64, or AES and NEON on ARM64. Check that your host or virtual machine +exposes these features before installing a wheel. + ## Installation ```bash From bb4a7d2f154178fb1d7fc9c0f64da1fe34dc8092 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 30 Oct 2026 12:55:59 -0500 Subject: [PATCH 14/17] Document the Windows Visual C++ runtime prerequisite --- docs/development/releasing-kumo-engine.md | 6 ++++++ packages/kumo-engine/README.md | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index 0bdf92e95..8a643cd3a 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -197,6 +197,12 @@ and takes the final platform tag from the built DiskGraph wheel. Rust, Xcode Command Line Tools, and CMake are required. The release workflow builds this wheel on a macOS ARM64 runner with the same deployment target. +Experimental native Windows x64 installations require a Microsoft Visual C++ +v14 runtime at least as recent as the build tools used for the wheel. If the +runtime is missing or too old, install the +[latest supported x64 Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). +The engine wheel does not bundle the redistributable. + For experimental native Windows builds, run the builder in Git Bash with native Windows Python, the Rust MSVC toolchain, CMake, and an activated Visual Studio x64 C++/Windows SDK environment. Set `PYTHON=python` if `python3` diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index c9ccbecde..6ce2418da 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -21,6 +21,11 @@ The bundled native engine requires CPU AES instructions: AES and SSE2 on x86-64, or AES and NEON on ARM64. Check that your host or virtual machine exposes these features before installing a wheel. +Native Windows x64 installations require the Microsoft Visual C++ v14 runtime +for the bundled ingestion executable. If it is missing, install the +[latest supported x64 Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). +Visual Studio build tools are needed only when building from source. + ## Installation ```bash From 0d43e01ef32748791030d656c891740afacda31c Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 30 Oct 2026 14:15:58 -0500 Subject: [PATCH 15/17] Limit Windows release support to WSL --- docs/development/releasing-kumo-engine.md | 55 +++++++++++++---------- packages/kumo-engine/README.md | 14 +++--- packages/kumo-engine/platform-smoke.md | 19 +++++--- 3 files changed, 51 insertions(+), 37 deletions(-) diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index 8a643cd3a..1546b5b1e 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -33,7 +33,8 @@ Faiss is selected at install time: `faiss-gpu==1.14.3` on Linux x86-64 and `faiss-cpu==1.14.3` on macOS. These markers live on the engine's direct dependencies so merging bundled packages preserves them. This dependency selection does not establish wheel or runtime support. Faiss is optional on -Windows and Linux ARM64; the `cpu` extra opts into CPU Faiss there. +Linux ARM64 and experimental native Windows builds; the `cpu` extra opts into +CPU Faiss there. Windows support for this release is WSL2 only. Releases are cut from `NVIDIA/kumo-engine`: its GitHub releases trigger the workflows below, which attach the wheels to the release. Their `kitmaker` job @@ -48,10 +49,10 @@ The build matrix targets `cp311-abi3-manylinux_2_34_x86_64` and WSL uses the Linux x86-64 wheel. All require CPython >= 3.11 (DiskGraph's stable-ABI floor), with classifiers for 3.11–3.14. Adding build targets does not establish published availability or installed-wheel runtime -qualification. Release attachment and publication currently allow only Linux -x86-64 and macOS ARM64. Linux ARM64 remains a CI artifact until installed-wheel -qualification is integrated; native Windows remains experimental and CI-only. -Windows release support uses WSL with the Linux x86-64 wheel. The merger reads +qualification. Native Windows builds and tests are experimental: their wheels +remain CI artifacts and are excluded from public release attachments and +Kitmaker publication. The supported Windows route for this release is WSL2. +The merger reads every bundled wheel's `Requires-Python` and writes their intersection into the final wheel. Stricter minimums, upper bounds, and exclusions are retained; incompatible @@ -124,17 +125,18 @@ platform's `release-version`. Labels and ranks can change only when their scores justify the change within that bound. The macOS wheel has no GPU smoke check. See the [test criteria](testing.md#finished-kumo-engine-wheel). - - attaches only the Linux x86-64 and macOS ARM64 wheels to the GitHub - release; ARM64 Linux and experimental native Windows builds stay in CI. + - attaches the Linux x86_64/ARM64 and macOS ARM64 wheels to the GitHub + release; the Linux x86_64 wheel also serves WSL2. Native Windows upload + routes remain disabled despite experimental build/test coverage. 5. **Publish through Kitmaker.** The `kitmaker` job runs in the `kumo-engine-release` environment and waits for approval from `kumo-engineers`. The person who starts the run cannot approve it. It checks that the Portal project has the expected name and owner and that the attached wheel is the one this run built, has Kitmaker dry-run it, and only then publishes it to PyPI and - pypi.nvidia.com, where it confirms both serve exactly those bytes. All enabled - platform wheels are published. A publication cannot be recalled, and PyPI never - accepts the same file name twice. + pypi.nvidia.com, where it confirms both serve exactly those bytes. All + enabled platform wheels are published. A publication cannot be recalled, + and PyPI never accepts the same file name twice. 6. **Smoke-test the published wheel** in a clean virtualenv: ```bash @@ -197,17 +199,23 @@ and takes the final platform tag from the built DiskGraph wheel. Rust, Xcode Command Line Tools, and CMake are required. The release workflow builds this wheel on a macOS ARM64 runner with the same deployment target. -Experimental native Windows x64 installations require a Microsoft Visual C++ -v14 runtime at least as recent as the build tools used for the wheel. If the -runtime is missing or too old, install the +### Experimental native Windows builds + +These instructions are for development and CI qualification, not a supported +installation route for this release. Use WSL2 and the Linux x86-64 wheel on +Windows. Native Windows release uploads are disabled. + +Experimental native Windows x64 installations require a Microsoft Visual C++ v14 runtime +at least as recent as the build tools used for the wheel. If the runtime is +missing or too old, install the [latest supported x64 Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). -The engine wheel does not bundle the redistributable. +This runtime prerequisite is separate from the MSVC build tools required to +build from source; the engine wheel does not bundle the redistributable. -For experimental native Windows builds, run the builder in Git Bash with -native Windows Python, the Rust MSVC toolchain, CMake, and an activated -Visual Studio x64 C++/Windows SDK environment. Set `PYTHON=python` if `python3` -is not available. Zig and ELF/glibc checks are disabled; the builder requires -DiskGraph's `win_amd64` +On native Windows, run the builder in Git Bash with native Windows Python, +the Rust MSVC toolchain, CMake, and an activated Visual Studio x64 C++/Windows +SDK environment. Set `PYTHON=python` if `python3` is not available. Zig and +ELF/glibc checks are disabled; the builder requires DiskGraph's `win_amd64` tag. `--verify-image` is not supported for Windows: qualify the installed wheel natively. The workflow activates the image's existing MSVC tools; it does not install system toolchains. @@ -217,12 +225,13 @@ hashes verify exact source bytes. For a fresh local clone, use only to the checkout action's Git processes, without changing repository or user Git configuration files or weakening the integrity checks. +### Linux ARM64 builds + Linux ARM64 uses the same Zig build and glibc checks as Linux x86-64, with `--platform-tag manylinux_2_34_aarch64`. The workflow validates the -assembled native payload before retaining it as a CI artifact. Clean -installed-wheel CPU ingestion, training, and prediction are a separate -qualification requirement; GPU execution is not required for these platform -wheels. +assembled native payload before uploading it. Clean installed-wheel CPU +ingestion, training, and prediction are a separate qualification requirement; +GPU execution is not required for these platform wheels. ## PyPI projects diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 6ce2418da..8006deed8 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -11,7 +11,8 @@ Kumo Engine (`kumo.engine`) trains graph neural networks on relational data, in- ## Requirements -- Linux x86_64 with glibc 2.34 or newer, or macOS 14 or newer on Apple silicon +- Linux x86_64 or ARM64 with glibc 2.34 or newer, or macOS 14 or newer on Apple silicon +- On Windows, use WSL2 x86_64 with a Linux distribution meeting that glibc requirement - CPython 3.11 or newer - For production workloads, an NVIDIA GPU with a compatible driver; see [prerequisites](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/prerequisites) and [GPU sizing](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/gpu-sizing) @@ -21,10 +22,9 @@ The bundled native engine requires CPU AES instructions: AES and SSE2 on x86-64, or AES and NEON on ARM64. Check that your host or virtual machine exposes these features before installing a wheel. -Native Windows x64 installations require the Microsoft Visual C++ v14 runtime -for the bundled ingestion executable. If it is missing, install the -[latest supported x64 Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). -Visual Studio build tools are needed only when building from source. +Windows support for this release is limited to WSL2 using the Linux x86_64 +wheel. Run installation and Python commands inside WSL. Native Windows is not +supported, and native Windows wheels are not published in this release. ## Installation @@ -34,8 +34,8 @@ pip install kumo-engine Linux x86_64 requires GPU Faiss, which also handles CPU retrieval. macOS keeps CPU Faiss and its existing Metal retrieval support. Faiss is optional on ARM64 -Linux and native Windows: without it, affected link-prediction retrieval raises -an installation error while other tasks remain available. On those platforms, +Linux: without it, affected link-prediction retrieval raises an installation +error while other tasks remain available. On ARM64 Linux, use `pip install "kumo-engine[cpu]"` for CPU retrieval with a compatible backend wheel for your platform and Python version. For CUDA retrieval, install a compatible GPU Faiss build directly, building from source on ARM64 Linux if diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index 2f230d061..aa79f8c4b 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -8,6 +8,11 @@ candidate branch. Manual runs upload artifacts; they do not publish a release. Qualification covers Python 3.11, 3.12, 3.13 and 3.14 on native Windows x86-64, Windows through WSL x86-64, and Linux ARM64: twelve independent CPU jobs. +Native Windows jobs provide experimental build and runtime evidence only. +For this release, Windows support is limited to WSL2; native Windows wheels +are retained as CI artifacts but excluded from public release attachments and +publication. A passing native job does not establish supported native Windows +distribution. WSL installs the same Linux x86-64 wheel as ordinary Linux; it is not a separate wheel format. GPU availability is not a qualification requirement. The producer also retains the macOS ARM64 artifact and its existing checks; @@ -37,13 +42,13 @@ compiler and rebuild no native artifacts: missing executables or libraries must fail the downloaded-wheel check. Logs, environment evidence and pytest results distinguish packaging failures from training failures. -Historical [source smoke runs](https://github.com/NVIDIA/kumo-engine/actions/runs/39667658574) -passed native Windows CPU and WSL CUDA ingestion/training/prediction. A local -Linux ARM64 development wheel (`4e910365`) passed clean Python 3.12 CPU -installation, dependency checks, origin auditing, and this smoke with four -optimizer steps and 100 predictions, without Faiss. These -results do not qualify the current candidate across the twelve installed-wheel -jobs; current qualification requires successful runs for that candidate. +The [installed-wheel qualification run](https://github.com/NVIDIA/kumo-engine/actions/runs/41101558418) +at `8f6c496f` passed all twelve jobs, each with four CPU optimizer steps and +100 finite predictions. Windows native and Linux ARM64 passed without Faiss. +This establishes runtime evidence for those artifacts, including experimental +native Windows coverage; it does not change the WSL-only Windows release +boundary or publish a package. Later runtime or packaging changes require +qualification of their candidate artifacts. PR checks remain a later decision after manual qualification is reliable. This change does not require an ARM GPU runner grant or change publication From 62beb3b0a2bd862a103c797e529ec723c2cc57f0 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Fri, 30 Oct 2026 14:19:16 -0500 Subject: [PATCH 16/17] Gate ARM64 release artifacts on CPU qualification --- .github/workflows/kumo-engine-wheel.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/kumo-engine-wheel.yml b/.github/workflows/kumo-engine-wheel.yml index e66a7d0b3..f6e899500 100644 --- a/.github/workflows/kumo-engine-wheel.yml +++ b/.github/workflows/kumo-engine-wheel.yml @@ -252,8 +252,7 @@ jobs: include: - arch: x86_64 - arch: arm64 - # Enabled by dependent qualification PR #92 with its runtime gate. - # - arch: aarch64 + - arch: aarch64 # Native Windows remains experimental; Windows releases use WSL. # - arch: windows-x86_64 runs-on: ubuntu-24.04 @@ -283,9 +282,8 @@ jobs: # Native Windows remains experimental; Windows releases use WSL. # - arch: windows-x86_64 # platform: win_amd64 - # Enabled by dependent qualification PR #92 with its runtime gate. - # - arch: aarch64 - # platform: manylinux_2_34_aarch64 + - arch: aarch64 + platform: manylinux_2_34_aarch64 - arch: x86_64 platform: manylinux_2_34_x86_64 - arch: arm64 From 8f5471712d5b36312b1ec899166da409ff9b1454 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Mon, 2 Nov 2026 13:26:27 -0600 Subject: [PATCH 17/17] Check saved model reload in platform wheel smoke --- packages/kumo-engine/platform-smoke.md | 7 +++- packages/kumo-engine/test/smoke_platform.py | 40 +++++++++++++-------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md index aa79f8c4b..65c67a60d 100644 --- a/packages/kumo-engine/platform-smoke.md +++ b/packages/kumo-engine/platform-smoke.md @@ -31,7 +31,12 @@ WSL retains the wheel's existing Linux x86-64 Faiss dependency. performs real snapshot ingestion, and selects CPU through the public `Trainer(..., accelerator='cpu')` API. Acceptance requires four optimizer steps with finite CPU gradients, finite training metrics, and all 100 expected -finite predictions. Synchronous DataLoader execution limits Windows coverage +finite predictions. It reloads the same fit's saved bundle with the public +`Trainer.load(..., accelerator='cpu')` API and predicts the same population and +snapshot. Reloaded entity IDs and counts must match, and prediction values must +match within `atol=1e-4` and NumPy's default relative tolerance. The optimizer +hook stays active throughout, requiring no additional training steps during +either prediction. Synchronous DataLoader execution limits Windows coverage to zero worker processes; Unix worker pools and native Windows GPU execution are outside this qualification. Subprocess diagnostics are printed only for this generated fixture, preserving production error privacy. diff --git a/packages/kumo-engine/test/smoke_platform.py b/packages/kumo-engine/test/smoke_platform.py index aa459437e..188b0c9f4 100644 --- a/packages/kumo-engine/test/smoke_platform.py +++ b/packages/kumo-engine/test/smoke_platform.py @@ -85,24 +85,36 @@ def record_optimizer_step(optimizer, _args, _kwargs): try: fitted = trainer.fit(query.generate_training_table(snapshot), snapshot=snapshot) + assert len(optimizer_steps) == 4, 'Expected four CPU optimizer steps' + assert fitted.metrics + assert all(math.isfinite(value) for value in fitted.metrics.values()) + assert any(name.startswith('train_') for name in fitted.metrics) + + population = query.generate_prediction_table(snapshot) + predictions = trainer.predict(population, snapshot=snapshot) + frame = predictions.frame.sort_values('ENTITY').reset_index(drop=True) + prediction_count = row_count - labeled_count + assert predictions.num_rows == prediction_count + assert len(frame) == prediction_count + assert set(frame['ENTITY']) == set(range(labeled_count, row_count)) + assert np.isfinite(frame['TARGET_PRED'].to_numpy()).all() + + restored = kumo.Trainer.load(fitted.bundle_dir, accelerator='cpu') + reloaded = restored.predict(population, snapshot=snapshot) + reloaded_frame = reloaded.frame.sort_values('ENTITY').reset_index( + drop=True) + assert reloaded.num_rows == prediction_count + assert len(reloaded_frame) == prediction_count + np.testing.assert_array_equal(reloaded_frame['ENTITY'], frame['ENTITY']) + assert np.isfinite(reloaded_frame['TARGET_PRED'].to_numpy()).all() + np.testing.assert_allclose(reloaded_frame['TARGET_PRED'], + frame['TARGET_PRED'], atol=1e-4) + assert len(optimizer_steps) == 4, 'Prediction unexpectedly trained' finally: hook.remove() - assert len(optimizer_steps) == 4, 'Expected four CPU optimizer steps' - assert fitted.metrics - assert all(math.isfinite(value) for value in fitted.metrics.values()) - assert any(name.startswith('train_') for name in fitted.metrics) - - predictions = trainer.predict( - query.generate_prediction_table(snapshot), snapshot=snapshot) - frame = predictions.frame - prediction_count = row_count - labeled_count - assert predictions.num_rows == prediction_count - assert len(frame) == prediction_count - assert set(frame['ENTITY']) == set(range(labeled_count, row_count)) - assert np.isfinite(frame['TARGET_PRED'].to_numpy()).all() print(f'Platform smoke completed: {len(optimizer_steps)} ' f'{device} optimizer steps, ' - f'{len(frame)} finite predictions', + f'{len(frame)} finite predictions and matching bundle reload', flush=True) except DiskgraphIngestError as error: # This test uses only generated fixture data; expose its subprocess