diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml new file mode 100644 index 000000000..fa0e8759e --- /dev/null +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: Kumo Engine - Platform wheel CPU qualification + +on: # yamllint disable-line rule:truthy + workflow_call: + +permissions: + contents: read + +jobs: + smoke: + name: CPU wheel (${{ matrix.target.platform }}, Python ${{ matrix.python }}) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12', '3.13', '3.14'] + target: + - platform: windows-native + runner: windows-2022 + artifact: windows-x86_64 + - platform: windows-wsl + runner: wsl-amd64-gpu-t4-latest-1 + artifact: x86_64 + - platform: linux-arm64 + runner: ubuntu-24.04-arm + artifact: aarch64 + runs-on: ${{ matrix.target.runner }} + defaults: + run: + shell: bash + env: + PYTHONUNBUFFERED: '1' + PYTHONPATH: '' + PYTEST_DISABLE_PLUGIN_AUTOLOAD: '1' + CUDA_VISIBLE_DEVICES: '' + UV_LINK_MODE: copy + UV_NO_CONFIG: 'true' + SMOKE_PLATFORM: ${{ matrix.target.platform }} + SMOKE_PYTHON: ${{ matrix.python }} + CANDIDATE_SHA: ${{ github.sha }} + steps: + - name: Checkout the candidate test harness + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Download the candidate wheel from this run + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: kumo-engine-wheel-${{ matrix.target.artifact }}-${{ github.sha }} + path: ${{ runner.temp }}/platform-wheel/artifacts + + - name: Set up uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + enable-cache: false + + - name: Clean install and qualify the finished wheel + run: | + set -euo pipefail + unset PYTHONPATH + stage="$RUNNER_TEMP/platform-wheel" + if [ "$RUNNER_OS" = Windows ]; then + stage="$(cygpath -u "$stage")" + fi + mkdir -p "$stage/results" "$stage/test" + cp packages/kumo-engine/test/smoke_platform.py "$stage/test/" + cp packages/kumo-engine/scripts/verify_installed_wheel.py "$stage/test/" + cd "$stage" + wheels=(artifacts/kumo_engine-*.whl) + if [ "${#wheels[@]}" -ne 1 ] || [ ! -f "${wheels[0]}" ]; then + echo 'Expected exactly one candidate engine wheel.' >&2 + exit 1 + fi + uv venv --python "$SMOKE_PYTHON" --seed venv + if [ "$RUNNER_OS" = Windows ]; then + python="$stage/venv/Scripts/python.exe" + else + python="$stage/venv/bin/python" + fi + uv pip install --python "$python" --torch-backend=cpu \ + "${wheels[0]}" pytest 2>&1 | tee results/install.log + uv pip check --python "$python" 2>&1 | tee results/dependencies.log + "$python" test/verify_installed_wheel.py \ + --wheel "${wheels[0]}" --platform "$SMOKE_PLATFORM" \ + --python-version "$SMOKE_PYTHON" --commit "$CANDIDATE_SHA" \ + --output results/environment.json + cd test + "$python" -m pytest -q -s --confcutdir=. \ + --junitxml=../results/junit.xml smoke_platform.py \ + 2>&1 | tee ../results/smoke.log + + - name: Save platform qualification evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: engine-wheel-cpu-${{ matrix.target.platform }}-py${{ matrix.python }}-${{ github.sha }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/platform-wheel/results + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/kumo-engine-wheel.yml b/.github/workflows/kumo-engine-wheel.yml index 7e70aeda2..f6e899500 100644 --- a/.github/workflows/kumo-engine-wheel.yml +++ b/.github/workflows/kumo-engine-wheel.yml @@ -239,8 +239,12 @@ jobs: with: wheel_artifact: kumo-engine-wheel-x86_64-${{ github.sha }} + platform-qualification: + needs: [build] + uses: ./.github/workflows/kumo-engine-platform-smoke.yml + attach-to-release: - needs: [build, check-requirements, wheel-smoke] + needs: [build, check-requirements, wheel-smoke, platform-qualification] if: ${{ github.event_name == 'release' }} strategy: fail-fast: false @@ -248,8 +252,7 @@ jobs: include: - arch: x86_64 - arch: arm64 - # Enabled by dependent qualification PR #92 with its runtime gate. - # - arch: aarch64 + - arch: aarch64 # Native Windows remains experimental; Windows releases use WSL. # - arch: windows-x86_64 runs-on: ubuntu-24.04 @@ -279,9 +282,8 @@ jobs: # Native Windows remains experimental; Windows releases use WSL. # - arch: windows-x86_64 # platform: win_amd64 - # Enabled by dependent qualification PR #92 with its runtime gate. - # - arch: aarch64 - # platform: manylinux_2_34_aarch64 + - arch: aarch64 + platform: manylinux_2_34_aarch64 - arch: x86_64 platform: manylinux_2_34_x86_64 - arch: arm64 diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index 25fcbab9f..1546b5b1e 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -33,7 +33,8 @@ Faiss is selected at install time: `faiss-gpu==1.14.3` on Linux x86-64 and `faiss-cpu==1.14.3` on macOS. These markers live on the engine's direct dependencies so merging bundled packages preserves them. This dependency selection does not establish wheel or runtime support. Faiss is optional on -Windows and Linux ARM64; the `cpu` extra opts into CPU Faiss there. +Linux ARM64 and experimental native Windows builds; the `cpu` extra opts into +CPU Faiss there. Windows support for this release is WSL2 only. Releases are cut from `NVIDIA/kumo-engine`: its GitHub releases trigger the workflows below, which attach the wheels to the release. Their `kitmaker` job @@ -48,10 +49,10 @@ The build matrix targets `cp311-abi3-manylinux_2_34_x86_64` and WSL uses the Linux x86-64 wheel. All require CPython >= 3.11 (DiskGraph's stable-ABI floor), with classifiers for 3.11–3.14. Adding build targets does not establish published availability or installed-wheel runtime -qualification. Release attachment and publication currently allow only Linux -x86-64 and macOS ARM64. Linux ARM64 remains a CI artifact until installed-wheel -qualification is integrated; native Windows remains experimental and CI-only. -Windows release support uses WSL with the Linux x86-64 wheel. The merger reads +qualification. Native Windows builds and tests are experimental: their wheels +remain CI artifacts and are excluded from public release attachments and +Kitmaker publication. The supported Windows route for this release is WSL2. +The merger reads every bundled wheel's `Requires-Python` and writes their intersection into the final wheel. Stricter minimums, upper bounds, and exclusions are retained; incompatible @@ -124,17 +125,18 @@ platform's `release-version`. Labels and ranks can change only when their scores justify the change within that bound. The macOS wheel has no GPU smoke check. See the [test criteria](testing.md#finished-kumo-engine-wheel). - - attaches only the Linux x86-64 and macOS ARM64 wheels to the GitHub - release; ARM64 Linux and experimental native Windows builds stay in CI. + - attaches the Linux x86_64/ARM64 and macOS ARM64 wheels to the GitHub + release; the Linux x86_64 wheel also serves WSL2. Native Windows upload + routes remain disabled despite experimental build/test coverage. 5. **Publish through Kitmaker.** The `kitmaker` job runs in the `kumo-engine-release` environment and waits for approval from `kumo-engineers`. The person who starts the run cannot approve it. It checks that the Portal project has the expected name and owner and that the attached wheel is the one this run built, has Kitmaker dry-run it, and only then publishes it to PyPI and - pypi.nvidia.com, where it confirms both serve exactly those bytes. All enabled - platform wheels are published. A publication cannot be recalled, and PyPI never - accepts the same file name twice. + pypi.nvidia.com, where it confirms both serve exactly those bytes. All + enabled platform wheels are published. A publication cannot be recalled, + and PyPI never accepts the same file name twice. 6. **Smoke-test the published wheel** in a clean virtualenv: ```bash @@ -169,6 +171,14 @@ as a workflow artifact for 30 days. It never attaches or publishes anything: attaching runs only for release events, and PyPI rejects local versions anyway. +## Native CPU baseline + +DiskGraph builds retain the fixed CPU baseline in `.cargo/config.toml` and +the native build backend: AES plus SSE2 on x86-64; AES plus NEON on ARM64. +There is no `target-cpu=native` setting or AVX2 requirement. Qualification +runners and deployment CPUs must expose that baseline, including through +virtualization. + ## Build locally ```bash @@ -189,11 +199,23 @@ and takes the final platform tag from the built DiskGraph wheel. Rust, Xcode Command Line Tools, and CMake are required. The release workflow builds this wheel on a macOS ARM64 runner with the same deployment target. -For experimental native Windows builds, run the builder in Git Bash with -native Windows Python, the Rust MSVC toolchain, CMake, and an activated -Visual Studio x64 C++/Windows SDK environment. Set `PYTHON=python` if `python3` -is not available. Zig and ELF/glibc checks are disabled; the builder requires -DiskGraph's `win_amd64` +### Experimental native Windows builds + +These instructions are for development and CI qualification, not a supported +installation route for this release. Use WSL2 and the Linux x86-64 wheel on +Windows. Native Windows release uploads are disabled. + +Experimental native Windows x64 installations require a Microsoft Visual C++ v14 runtime +at least as recent as the build tools used for the wheel. If the runtime is +missing or too old, install the +[latest supported x64 Redistributable](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist). +This runtime prerequisite is separate from the MSVC build tools required to +build from source; the engine wheel does not bundle the redistributable. + +On native Windows, run the builder in Git Bash with native Windows Python, +the Rust MSVC toolchain, CMake, and an activated Visual Studio x64 C++/Windows +SDK environment. Set `PYTHON=python` if `python3` is not available. Zig and +ELF/glibc checks are disabled; the builder requires DiskGraph's `win_amd64` tag. `--verify-image` is not supported for Windows: qualify the installed wheel natively. The workflow activates the image's existing MSVC tools; it does not install system toolchains. @@ -203,12 +225,13 @@ hashes verify exact source bytes. For a fresh local clone, use only to the checkout action's Git processes, without changing repository or user Git configuration files or weakening the integrity checks. +### Linux ARM64 builds + Linux ARM64 uses the same Zig build and glibc checks as Linux x86-64, with `--platform-tag manylinux_2_34_aarch64`. The workflow validates the -assembled native payload before retaining it as a CI artifact. Clean -installed-wheel CPU ingestion, training, and prediction are a separate -qualification requirement; GPU execution is not required for these platform -wheels. +assembled native payload before uploading it. Clean installed-wheel CPU +ingestion, training, and prediction are a separate qualification requirement; +GPU execution is not required for these platform wheels. ## PyPI projects diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index e425c3bbf..8006deed8 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -11,12 +11,21 @@ Kumo Engine (`kumo.engine`) trains graph neural networks on relational data, in- ## Requirements -- Linux x86_64 with glibc 2.34 or newer, or macOS 14 or newer on Apple silicon +- Linux x86_64 or ARM64 with glibc 2.34 or newer, or macOS 14 or newer on Apple silicon +- On Windows, use WSL2 x86_64 with a Linux distribution meeting that glibc requirement - CPython 3.11 or newer - For production workloads, an NVIDIA GPU with a compatible driver; see [prerequisites](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/prerequisites) and [GPU sizing](https://docs.nvidia.com/sdgm/kumo-engine/get-started-with-kumo-engine/gpu-sizing) Kumo Engine uses a GPU when one is available: an NVIDIA GPU through CUDA, or an Apple silicon GPU through PyTorch's MPS backend. Without one, training and prediction run on CPU, which is enough for small jobs such as the quickstart below. To choose the device, pass `accelerator=` (`"auto"`, `"cpu"`, `"gpu"`, or `"mps"`) to `Trainer`. +The bundled native engine requires CPU AES instructions: AES and SSE2 on +x86-64, or AES and NEON on ARM64. Check that your host or virtual machine +exposes these features before installing a wheel. + +Windows support for this release is limited to WSL2 using the Linux x86_64 +wheel. Run installation and Python commands inside WSL. Native Windows is not +supported, and native Windows wheels are not published in this release. + ## Installation ```bash @@ -25,8 +34,8 @@ pip install kumo-engine Linux x86_64 requires GPU Faiss, which also handles CPU retrieval. macOS keeps CPU Faiss and its existing Metal retrieval support. Faiss is optional on ARM64 -Linux and native Windows: without it, affected link-prediction retrieval raises -an installation error while other tasks remain available. On those platforms, +Linux: without it, affected link-prediction retrieval raises an installation +error while other tasks remain available. On ARM64 Linux, use `pip install "kumo-engine[cpu]"` for CPU retrieval with a compatible backend wheel for your platform and Python version. For CUDA retrieval, install a compatible GPU Faiss build directly, building from source on ARM64 Linux if diff --git a/packages/kumo-engine/platform-smoke.md b/packages/kumo-engine/platform-smoke.md new file mode 100644 index 000000000..65c67a60d --- /dev/null +++ b/packages/kumo-engine/platform-smoke.md @@ -0,0 +1,60 @@ +# Installed-wheel platform qualification + +The manual [wheel build workflow](../../.github/workflows/kumo-engine-wheel.yml) +builds the candidate once and passes those artifacts to the reusable +[platform smoke workflow](../../.github/workflows/kumo-engine-platform-smoke.yml). +Select **Kumo Engine - Build and publish wheel**, **Run workflow**, and the +candidate branch. Manual runs upload artifacts; they do not publish a release. + +Qualification covers Python 3.11, 3.12, 3.13 and 3.14 on native Windows x86-64, +Windows through WSL x86-64, and Linux ARM64: twelve independent CPU jobs. +Native Windows jobs provide experimental build and runtime evidence only. +For this release, Windows support is limited to WSL2; native Windows wheels +are retained as CI artifacts but excluded from public release attachments and +publication. A passing native job does not establish supported native Windows +distribution. +WSL installs the same Linux x86-64 wheel as ordinary Linux; it is not a +separate wheel format. GPU availability is not a qualification requirement. +The producer also retains the macOS ARM64 artifact and its existing checks; +this smoke does not add macOS runtime qualification. + +Each consumer creates a fresh virtual environment, installs the downloaded +wheel and public CPU Torch dependencies, and runs dependency consistency +checks. It copies only the smoke test and standalone environment auditor +outside the checkout. The auditor rejects checkout imports, records origins +for engine/core/API/PQL/DiskGraph, verifies the packaged ingestion executable, +and records the wheel filename, SHA-256, candidate revision, Python, platform, +and Torch version. Native Windows and Linux ARM64 must run without Faiss; +WSL retains the wheel's existing Linux x86-64 Faiss dependency. + +[smoke_platform.py](test/smoke_platform.py) creates a synthetic Parquet table, +performs real snapshot ingestion, and selects CPU through the public +`Trainer(..., accelerator='cpu')` API. Acceptance requires four optimizer +steps with finite CPU gradients, finite training metrics, and all 100 expected +finite predictions. It reloads the same fit's saved bundle with the public +`Trainer.load(..., accelerator='cpu')` API and predicts the same population and +snapshot. Reloaded entity IDs and counts must match, and prediction values must +match within `atol=1e-4` and NumPy's default relative tolerance. The optimizer +hook stays active throughout, requiring no additional training steps during +either prediction. Synchronous DataLoader execution limits Windows coverage +to zero worker processes; Unix worker pools and native Windows GPU execution +are outside this qualification. Subprocess diagnostics are printed only for +this generated fixture, preserving production error privacy. + +The WSL job checks the Linux guest kernel. A Windows host shell or ordinary +Linux x86-64 runner cannot substitute for WSL. Consumer jobs install no Rust +compiler and rebuild no native artifacts: missing executables or libraries +must fail the downloaded-wheel check. Logs, environment evidence and pytest +results distinguish packaging failures from training failures. + +The [installed-wheel qualification run](https://github.com/NVIDIA/kumo-engine/actions/runs/41101558418) +at `8f6c496f` passed all twelve jobs, each with four CPU optimizer steps and +100 finite predictions. Windows native and Linux ARM64 passed without Faiss. +This establishes runtime evidence for those artifacts, including experimental +native Windows coverage; it does not change the WSL-only Windows release +boundary or publish a package. Later runtime or packaging changes require +qualification of their candidate artifacts. + +PR checks remain a later decision after manual qualification is reliable. +This change does not require an ARM GPU runner grant or change publication +approval policy. diff --git a/packages/kumo-engine/scripts/verify_installed_wheel.py b/packages/kumo-engine/scripts/verify_installed_wheel.py new file mode 100644 index 000000000..e74af30b3 --- /dev/null +++ b/packages/kumo-engine/scripts/verify_installed_wheel.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Record installed-wheel evidence before executing the CPU smoke. + +This standalone audit uses stdlib filesystem operations so it can reject +checkout imports before loading Kumo's own filesystem abstraction. +""" +from __future__ import annotations + +import argparse +import hashlib +import importlib +import importlib.metadata +import importlib.util +import json +import os +import platform +import sys +from pathlib import Path +from urllib.parse import urlsplit +from urllib.request import url2pathname + +MODULES = ('kumo.engine', 'kumo_core', 'kumoapi', 'kumopql', 'diskgraph._c') + + +def check_origin(origin: str | None, prefix: Path) -> str: + if origin is None: + raise ValueError('Required module has no installed file origin') + path = Path(origin).resolve() + if not path.is_relative_to(prefix.resolve()): + raise ValueError(f'Module originates outside the virtual environment: {path}') + return str(path) + + +def check_platform(expected: str, system: str, machine: str, release: str) -> None: + arm = machine.lower() in {'aarch64', 'arm64'} + amd = machine.lower() in {'amd64', 'x86_64'} + wsl = 'microsoft' in release.lower() + valid = { + 'windows-native': system == 'Windows' and amd, + 'windows-wsl': system == 'Linux' and amd and wsl, + 'linux-arm64': system == 'Linux' and arm and not wsl, + } + if not valid[expected]: + raise ValueError(f'Unexpected runtime: {system}/{machine}/{release}') + + +def check_install(source: str | None, wheel: Path) -> None: + if source is None: + raise ValueError('Installed distribution lacks direct-wheel provenance') + recorded = urlsplit(json.loads(source)['url']) + if (recorded.scheme != 'file' or recorded.netloc + or recorded.query or recorded.fragment): + raise ValueError('Installed distribution is not from a local wheel') + installed_path = Path(url2pathname(recorded.path)).resolve() + if os.path.normcase(str(installed_path)) != os.path.normcase(str(wheel.resolve())): + raise ValueError('Installed distribution differs from the downloaded wheel') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--wheel', required=True, type=Path) + parser.add_argument('--platform', required=True, + choices=('windows-native', 'windows-wsl', 'linux-arm64')) + parser.add_argument('--python-version', required=True) + parser.add_argument('--commit', required=True) + parser.add_argument('--output', required=True, type=Path) + args = parser.parse_args() + if sys.prefix == sys.base_prefix: + raise ValueError('Qualification requires a fresh virtual environment') + if os.environ.get('PYTHONPATH') or os.environ.get('DISKGRAPH_INGEST_BIN'): + raise ValueError('PYTHONPATH and DISKGRAPH_INGEST_BIN must be unset') + actual_python = f'{sys.version_info.major}.{sys.version_info.minor}' + if actual_python != args.python_version: + raise ValueError('Python version differs from the qualification matrix') + check_platform(args.platform, platform.system(), platform.machine(), + platform.release()) + origins = {name: check_origin(importlib.import_module(name).__file__, + Path(sys.prefix)) for name in MODULES} + diskgraph = importlib.import_module('diskgraph') + package = Path(check_origin(diskgraph.__file__, Path(sys.prefix))).parent + ingest = package / 'bin' / ('ingest.exe' if sys.platform == 'win32' else 'ingest') + if not ingest.is_file() or not os.access(ingest, os.X_OK): + raise ValueError('Installed wheel ingestion executable is missing') + faiss_present = importlib.util.find_spec('faiss') is not None + if args.platform != 'windows-wsl' and faiss_present: + raise ValueError('Bare Windows/ARM wheel qualification must omit Faiss') + import torch + with args.wheel.open('rb') as wheel_file: + wheel_sha256 = hashlib.file_digest(wheel_file, 'sha256').hexdigest() + check_install(importlib.metadata.distribution('kumo-engine').read_text( + 'direct_url.json'), args.wheel) + report = { + 'commit': args.commit, 'wheel': args.wheel.name, + 'wheel_sha256': wheel_sha256, + 'platform': args.platform, 'system': platform.platform(), + 'python': sys.version, 'executable': sys.executable, + 'origins': origins, 'ingest': str(ingest), 'faiss_present': faiss_present, + 'torch': torch.__version__, 'torch_cuda_build': torch.version.cuda, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report, sort_keys=True), flush=True) + + +if __name__ == '__main__': + main() diff --git a/packages/kumo-engine/test/smoke_platform.py b/packages/kumo-engine/test/smoke_platform.py new file mode 100644 index 000000000..188b0c9f4 --- /dev/null +++ b/packages/kumo-engine/test/smoke_platform.py @@ -0,0 +1,125 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Explicit smoke: pytest -q -s packages/kumo-engine/test/smoke_platform.py. + +Run only against an installed wheel outside the source checkout. +CPU training uses the public Trainer accelerator option. +The filename keeps this integration check out of ordinary test discovery. +""" + +import json +import math +import platform +import sys + +import numpy as np +import pandas as pd +import torch +from kumo_core.diskgraph.runner import DiskgraphIngestError +from kumoapi.model_plan import ModelPlan +from torch.optim.optimizer import register_optimizer_step_post_hook + +import kumo.engine as kumo +from kumo.engine._scratch import get_scratch_dir + + +def test_platform_train_predict(tmp_path, tmp_path_factory, monkeypatch): + device = 'cpu' + monkeypatch.setenv('SYNC_DATA_LOADER', '1') + diagnostics = { + 'requested_device': device, + 'platform': platform.platform(), + 'architecture': platform.machine(), + 'python': sys.version, + 'torch': torch.__version__, + 'cuda_runtime': torch.version.cuda, + 'cuda_available': torch.cuda.is_available(), + } + print(json.dumps(diagnostics, sort_keys=True), flush=True) + + previous = get_scratch_dir() + kumo.configure(scratch_dir=tmp_path / 'scratch') + try: + # A single numerical table exercises real ingestion and training + # without external datasets, credentials, or text-model downloads. + rng = np.random.default_rng(0) + row_count = 400 + labeled_count = 300 + targets = rng.random(row_count) + targets[labeled_count:] = np.nan + source_dir = tmp_path_factory.mktemp('smoke-source') + pd.DataFrame({ + 'ID': np.arange(row_count), + 'Y': targets, + 'F1': rng.normal(size=row_count), + }).to_parquet(source_dir / 'entities.parquet') + connector = kumo.FileConnector(root_dir=str(source_dir)) + entities = kumo.Table.from_source_table( + connector['entities.parquet'], primary_key='ID').infer_metadata() + graph = kumo.Graph(tables={'entities': entities}, edges=[]) + snapshot = graph.snapshot() + query = kumo.PredictiveQuery( + graph=graph, query='PREDICT entities.Y FOR EACH entities.ID') + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.disable_explain = True + plan.optimization.max_epochs = 1 + plan.optimization.max_steps_per_epoch = 2 + trainer = kumo.Trainer(model_plan=plan, accelerator='cpu') + optimizer_steps = [] + + def record_optimizer_step(optimizer, _args, _kwargs): + gradients = [parameter.grad for group in optimizer.param_groups + for parameter in group['params'] + if parameter.grad is not None] + assert gradients, 'Optimizer step did not produce gradients' + assert all(gradient.device.type == device for gradient in gradients) + assert all(torch.isfinite(gradient).all().item() + for gradient in gradients) + optimizer_steps.append(1) + + # Observe real optimizer work; availability alone cannot establish + # that training actually ran on the requested device. + hook = register_optimizer_step_post_hook(record_optimizer_step) + try: + fitted = trainer.fit(query.generate_training_table(snapshot), + snapshot=snapshot) + assert len(optimizer_steps) == 4, 'Expected four CPU optimizer steps' + assert fitted.metrics + assert all(math.isfinite(value) for value in fitted.metrics.values()) + assert any(name.startswith('train_') for name in fitted.metrics) + + population = query.generate_prediction_table(snapshot) + predictions = trainer.predict(population, snapshot=snapshot) + frame = predictions.frame.sort_values('ENTITY').reset_index(drop=True) + prediction_count = row_count - labeled_count + assert predictions.num_rows == prediction_count + assert len(frame) == prediction_count + assert set(frame['ENTITY']) == set(range(labeled_count, row_count)) + assert np.isfinite(frame['TARGET_PRED'].to_numpy()).all() + + restored = kumo.Trainer.load(fitted.bundle_dir, accelerator='cpu') + reloaded = restored.predict(population, snapshot=snapshot) + reloaded_frame = reloaded.frame.sort_values('ENTITY').reset_index( + drop=True) + assert reloaded.num_rows == prediction_count + assert len(reloaded_frame) == prediction_count + np.testing.assert_array_equal(reloaded_frame['ENTITY'], frame['ENTITY']) + assert np.isfinite(reloaded_frame['TARGET_PRED'].to_numpy()).all() + np.testing.assert_allclose(reloaded_frame['TARGET_PRED'], + frame['TARGET_PRED'], atol=1e-4) + assert len(optimizer_steps) == 4, 'Prediction unexpectedly trained' + finally: + hook.remove() + print(f'Platform smoke completed: {len(optimizer_steps)} ' + f'{device} optimizer steps, ' + f'{len(frame)} finite predictions and matching bundle reload', + flush=True) + except DiskgraphIngestError as error: + # This test uses only generated fixture data; expose its subprocess + # diagnostics without changing production error privacy or retrying. + print(error.private_output_tail, file=sys.stderr, flush=True) + raise + finally: + kumo.configure(scratch_dir=previous) diff --git a/test/scripts/test_engine_wheel_workflow.py b/test/scripts/test_engine_wheel_workflow.py index 9999da208..f6e486ca1 100644 --- a/test/scripts/test_engine_wheel_workflow.py +++ b/test/scripts/test_engine_wheel_workflow.py @@ -147,8 +147,8 @@ def test_all_platforms_require_validation_and_release_environment(): assert 'pattern' not in download assert download['name'] == _step('build', 'Upload wheel')['with']['name'] assert JOBS['check-requirements']['needs'] == ['build'] - assert set(JOBS['attach-to-release']['needs']) >= { - 'build', 'check-requirements', 'wheel-smoke' + assert set(JOBS['attach-to-release']['needs']) == { + 'build', 'check-requirements', 'wheel-smoke', 'platform-qualification' } assert JOBS['kitmaker']['needs'] == ['attach-to-release'] assert JOBS['kitmaker']['environment'] == 'kumo-engine-release' @@ -180,3 +180,37 @@ def test_publication_forwards_the_shared_prediction_tolerance(): assert call['reload-atol'] == pr_call['reload-atol'] == '0.005' assert run['env']['RELOAD_ATOL'] == '${{ inputs.reload-atol }}' assert '--reload-atol "$RELOAD_ATOL"' in run['run'] + + +def test_cpu_qualification_uses_candidate_artifacts_on_all_pythons(): + caller = JOBS['platform-qualification'] + assert caller['needs'] == ['build'] + assert caller['uses'] == ( + './.github/workflows/kumo-engine-platform-smoke.yml') + path = WORKFLOW.parent / 'kumo-engine-platform-smoke.yml' + workflow = yaml.safe_load(path.read_text()) + # PyYAML's YAML 1.1 parser interprets the GitHub Actions `on` key as True. + assert workflow[True] == {'workflow_call': None} + job = workflow['jobs']['smoke'] + matrix = job['strategy']['matrix'] + assert set(matrix['python']) == {'3.11', '3.12', '3.13', '3.14'} + assert len(matrix['target']) == 3 + assert {row['platform']: row['artifact'] for row in matrix['target']} == { + 'windows-native': 'windows-x86_64', 'windows-wsl': 'x86_64', + 'linux-arm64': 'aarch64', + } + assert {row['artifact'] for row in matrix['target']} <= { + build['arch'] for build in BUILDS} + assert job['env']['CUDA_VISIBLE_DEVICES'] == '' + download = next(step for step in job['steps'] + if step.get('uses', '').startswith( + 'actions/download-artifact@')) + assert download['with'] == { + 'name': ('kumo-engine-wheel-${{ matrix.target.artifact }}-' + '${{ github.sha }}'), + 'path': '${{ runner.temp }}/platform-wheel/artifacts', + } + # Supplying another run or token here could accidentally qualify a stale + # artifact while publishing the current build. + assert 'run-id' not in download['with'] + assert 'github-token' not in download['with'] diff --git a/test/scripts/test_installed_wheel.py b/test/scripts/test_installed_wheel.py new file mode 100644 index 000000000..e2e15b744 --- /dev/null +++ b/test/scripts/test_installed_wheel.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import json +import runpy +from pathlib import Path + +import pytest + +SCRIPT = (Path(__file__).resolve().parents[2] / + 'packages/kumo-engine/scripts/verify_installed_wheel.py') +HELPER = runpy.run_path(str(SCRIPT)) + + +def test_checkout_origin_rejected(tmp_path): + with pytest.raises(ValueError, match='outside'): + HELPER['check_origin'](str(tmp_path / 'checkout/module.py'), + tmp_path / 'venv') + + +def test_installed_origin_accepted(tmp_path): + origin = tmp_path / 'venv/site-packages/module.py' + assert HELPER['check_origin'](str(origin), tmp_path / 'venv') == str(origin) + + +@pytest.mark.parametrize('expected,system,machine,release', [ + ('windows-native', 'Windows', 'AMD64', '2022'), + ('windows-wsl', 'Linux', 'x86_64', '6.6-microsoft-standard-WSL2'), + ('linux-arm64', 'Linux', 'aarch64', '6.8'), +]) +def test_platform_accepted(expected, system, machine, release): + HELPER['check_platform'](expected, system, machine, release) + + +@pytest.mark.parametrize('system,machine,release', [ + ('Windows', 'AMD64', '2022'), + ('Linux', 'x86_64', '6.8'), + ('Linux', 'aarch64', '6.6-microsoft-standard-WSL2'), +]) +def test_wsl_requires_linux_guest(system, machine, release): + with pytest.raises(ValueError, match='Unexpected runtime'): + HELPER['check_platform']('windows-wsl', system, machine, release) + + +def test_install_matches_artifact(tmp_path): + wheel = tmp_path / 'engine+dev.whl' + source = json.dumps({'url': wheel.as_uri(), 'archive_info': {}}) + HELPER['check_install'](source, wheel) + HELPER['check_install'](source.replace('%2B', '+'), wheel) + with pytest.raises(ValueError, match='differs'): + HELPER['check_install'](source, tmp_path / 'other.whl') + + +def test_install_requires_provenance(tmp_path): + with pytest.raises(ValueError, match='provenance'): + HELPER['check_install'](None, tmp_path / 'engine.whl') + + +def test_install_rejects_remote_artifact(tmp_path): + with pytest.raises(ValueError, match='not from a local'): + HELPER['check_install']('{"url":"https://example.com/engine.whl"}', + tmp_path / 'engine.whl')