Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions .github/workflows/kumo-engine-platform-smoke.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,85 @@ permissions:
contents: read

jobs:
windows-reload:
name: Native Windows CPU reload (Python ${{ matrix.python }})
runs-on: windows-2022
timeout-minutes: 60
environment: {name: gha-kumo-workflow-tests, deployment: false}
permissions: {contents: read, id-token: write}
strategy:
fail-fast: false
matrix:
python: ['3.11', '3.12', '3.13', '3.14']
defaults: {run: {shell: bash}}
env:
PYTHONPATH: ''
UV_LINK_MODE: copy
UV_NO_CONFIG: 'true'
CANDIDATE_SHA: ${{ github.sha }}
steps:
- uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
with: {ref: '${{ github.sha }}', persist-credentials: false}
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8
with:
name: kumo-engine-wheel-windows-x86_64-${{ github.sha }}
path: ${{ runner.temp }}/windows-reload/artifacts
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with: {enable-cache: false}
- id: aws
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests
aws-region: us-west-2
output-env-credentials: false
output-credentials: true
unset-current-credentials: true
- name: Download and check the pinned PR172 inputs
env:
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
run: |
uv run --no-project --python 3.12 --with 'boto3>=1.34,<2' python -I - <<'PY'
import boto3, hashlib, json, os, tarfile
from pathlib import Path
spec = json.loads(Path('packages/kumo-engine/test/pr172/inputs.json').read_text())
archive = Path(os.environ['RUNNER_TEMP'], 'reload-inputs.tar.gz')
boto3.client('s3', region_name=spec['region']).download_file(spec['bucket'], spec['key'], str(archive))
if hashlib.sha256(archive.read_bytes()).hexdigest() != spec['sha256']:
raise SystemExit('PR172 input checksum differs')
with tarfile.open(archive) as source:
source.extractall(Path(os.environ['RUNNER_TEMP'], 'reload-inputs'), filter='data')
PY
- name: Qualify the installed Windows wheel in fresh processes
run: |
set -euo pipefail
[ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || exec env -u ACTIONS_ID_TOKEN_REQUEST_URL -u ACTIONS_ID_TOKEN_REQUEST_TOKEN bash --noprofile --norc -eo pipefail "$0"
stage="$(cygpath -u "$RUNNER_TEMP")/windows-reload"
inputs="$(cygpath -u "$RUNNER_TEMP")/reload-inputs"
mkdir -p "$stage/test" "$stage/results"
cp packages/kumo-engine/test/{desktop_reload_qualification,wheel_gpu_smoke}.py "$stage/test/"
cd "$stage"
wheels=(artifacts/kumo_engine-*.whl)
[ "${#wheels[@]}" -eq 1 ] && [ -f "${wheels[0]}" ]
uv venv --python '${{ matrix.python }}' venv
python="$stage/venv/Scripts/python.exe"
uv pip install --python "$python" --torch-backend=cpu "${wheels[0]}[cpu]" 2>&1 | tee results/install.log
uv pip check --python "$python" 2>&1 | tee results/dependencies.log
uv pip freeze --python "$python" > results/freeze.txt
"$python" -I test/desktop_reload_qualification.py run --wheel "${wheels[0]}" --device cpu \
--data-dir "$inputs" --output-dir results --commit "$CANDIDATE_SHA"
- if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
with:
name: engine-wheel-reload-windows-native-cpu-py${{ matrix.python }}-${{ github.sha }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/windows-reload/results
!${{ runner.temp }}/windows-reload/results/**/*.parquet
!${{ runner.temp }}/windows-reload/results/*/*.json
if-no-files-found: warn
retention-days: 14

smoke:
name: CPU wheel (${{ matrix.target.platform }}, Python ${{ matrix.python }})
timeout-minutes: 60
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/kumo-engine-wheel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,17 @@ jobs:
test --manifest-path packages/diskgraph/Cargo.toml \
--release --locked -p diskgraph-ingest --lib \
rejects_windows_profiling_before_output_changes
cargo --config packages/diskgraph/.cargo/config.toml \
test --manifest-path packages/diskgraph/Cargo.toml \
--release --locked -p diskgraph-index --lib \
graph::scratch_cleanup_tests::sync_directory_flushes_completed_entry \
-- --exact | tee directory-sync-tests.log
grep -Eq '^test result: ok\. 1 passed;' directory-sync-tests.log
cargo --config packages/diskgraph/.cargo/config.toml \
test --manifest-path packages/diskgraph/Cargo.toml \
--release --locked -p diskgraph-ingest --lib \
snapshot_mirror::tests | tee snapshot-mirror-tests.log
grep -Eq '^test result: ok\. [1-9][0-9]* passed;' snapshot-mirror-tests.log

# GitHub-hosted artifact storage, not S3: no AWS role to provision.
# Native Windows wheels are experimental internal CI artifacts only.
Expand Down Expand Up @@ -270,6 +281,9 @@ jobs:

platform-qualification:
needs: [build]
permissions:
contents: read
id-token: write
uses: ./.github/workflows/kumo-engine-platform-smoke.yml

attach-to-release:
Expand Down
9 changes: 9 additions & 0 deletions docs/development/releasing-kumo-engine.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,15 @@ targets does not establish published availability or installed-wheel runtime
qualification. Native Windows builds and tests are experimental: their wheels
remain CI artifacts and are excluded from public release attachments and
Kitmaker publication. The supported Windows route for this release is WSL2.
Every wheel build also qualifies the native Windows CPU wheel on Python
3.11–3.14 in fresh producer and consumer processes, with the `cpu` extra for
Faiss retrieval. This check is a prerequisite for release attachment; it does
not enable Windows publication. It exports and restores both models and
snapshots after deleting the producer's temporary storage, and checks ordinary
loader defaults without CI or synchronous-loader overrides. Results are saved
in `engine-wheel-reload-windows-native-cpu-*` artifacts. Native Windows CUDA
is not qualified by this check. The GitHub-hosted Windows account may have
administrator privileges; this is not restricted-account qualification.
The merger reads
every bundled wheel's `Requires-Python` and writes their intersection into the
final wheel.
Expand Down
67 changes: 49 additions & 18 deletions packages/kumo-engine/test/desktop_reload_qualification.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import platform
import random
import re
import resource
import shutil
import signal
import subprocess
Expand All @@ -29,6 +28,9 @@
from urllib.parse import urlsplit
from urllib.request import url2pathname

if sys.platform != 'win32':
import resource

GPU_SMOKE = Path(__file__).with_name('wheel_gpu_smoke.py')
SPEC = util.spec_from_file_location(GPU_SMOKE.stem, GPU_SMOKE)
tasks = util.module_from_spec(SPEC)
Expand All @@ -53,6 +55,8 @@ def label(device):
"""Name the OS or macOS version, CPU, device and Python of this run."""
if sys.platform == 'darwin':
host = 'macos-' + platform.mac_ver()[0].split('.')[0]
elif sys.platform == 'win32':
host = 'windows-native'
elif 'microsoft' in platform.release().lower():
host = 'windows-wsl'
else:
Expand Down Expand Up @@ -187,30 +191,56 @@ def worker(phase, task, device, durable, out, base, data):
out.with_suffix('.json').write_text(json.dumps(meta, default=sorted))


def worker_environment(base, device, extra):
"""Give each process fresh caches and ordinary-user loader defaults."""
drop = ('KUMO_', 'DISKGRAPH_')
masked = {'PYTEST_CURRENT_TEST', 'GITHUB_ACTIONS', 'SYNC_DATA_LOADER', 'CI'}
env = {k: v for k, v in os.environ.items()
if not k.startswith(drop) and k not in masked}
env.update(extra, HOME=str(base / 'home'), USERPROFILE=str(base / 'home'),
TMPDIR=str(base / 'tmp'), TEMP=str(base / 'tmp'),
TMP=str(base / 'tmp'), XDG_CACHE_HOME=str(base / 'cache'),
PYTORCH_ENABLE_MPS_FALLBACK='0')
if device == 'cpu':
env['CUDA_VISIBLE_DEVICES'] = ''
return env


def finish_process(process):
"""Terminate timed-out workers and wait before deleting their storage."""
if sys.platform == 'win32':
if process.poll() is None:
try:
subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'],
check=True, capture_output=True)
except subprocess.CalledProcessError:
if process.poll() is None:
raise
else:
with suppress(ProcessLookupError):
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=30)


def spawn(phase, task, device, durable, out, data=None, **extra):
"""Run one worker in a new process. Then kill its process group and
delete its scratch, HOME, TMPDIR, XDG cache, cwd and fixture copy."""
base = Path(tempfile.mkdtemp(prefix='drq-', dir='/tmp'))
"""Run and reap a worker, terminating its process tree on timeout.

Delete its scratch, HOME, TMPDIR, XDG cache, cwd and fixture copy.
"""
base = Path(tempfile.mkdtemp(prefix='drq-'))
for name in ('home', 'cache', 'cwd', 'tmp', 'scratch'):
(base / name).mkdir()
drop = ('KUMO_', 'DISKGRAPH_') # Could point at other binaries or data.
env = {k: v for k, v in os.environ.items() if not k.startswith(drop)}
env.update(extra, HOME=str(base / 'home'), TMPDIR=str(base / 'tmp'),
XDG_CACHE_HOME=str(base / 'cache'),
PYTORCH_ENABLE_MPS_FALLBACK='0') # No silent CPU fallback.
if device == 'cpu': # A CPU row must not reach a GPU.
env['CUDA_VISIBLE_DEVICES'] = ''
env = worker_environment(base, device, extra)
command = [sys.executable, '-Iu', __file__, 'worker', phase, task, device]
command += map(str, (durable, out, base, data))
with open(out.with_suffix('.log'), 'wb') as log:
process = subprocess.Popen(command, cwd=base / 'cwd', env=env,
stdout=log, stderr=subprocess.STDOUT,
start_new_session=True)
start_new_session=sys.platform != 'win32')
try:
return process.wait(timeout=3600)
finally:
with suppress(ProcessLookupError):
os.killpg(process.pid, signal.SIGKILL)
finish_process(process)
shutil.rmtree(base)


Expand Down Expand Up @@ -296,7 +326,7 @@ def main():
summary = dict(label=label(args.device), commit=args.commit, started=now,
wheel=args.wheel.name, wheel_sha256=None,
cpus=os.cpu_count(), tasks={})
work = Path(tempfile.mkdtemp(prefix='drq-work-', dir='/tmp'))
work = Path(tempfile.mkdtemp(prefix='drq-work-'))
try:
for task in TASKS:
started = time.monotonic()
Expand All @@ -313,9 +343,10 @@ def main():
error=f'{type(error).__name__}: {error}')
result['seconds'] = round(time.monotonic() - started, 1)
# The largest worker's ru_maxrss: KiB on Linux, bytes on macOS.
rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
summary['peak_rss_mib'] = round(
rss / (1 << 20 if sys.platform == 'darwin' else 1 << 10))
if sys.platform != 'win32':
rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
summary['peak_rss_mib'] = round(
rss / (1 << 20 if sys.platform == 'darwin' else 1 << 10))
print(task, json.dumps(result), flush=True)
(output / 'summary.json').write_text(json.dumps(summary, indent=1))
finally:
Expand Down
105 changes: 105 additions & 0 deletions test/scripts/test_desktop_reload_qualification.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

import importlib.util
import json
import os
import subprocess
import sys
from pathlib import Path
from unittest.mock import Mock

import pytest

from io_utils import fsutil

SCRIPT = (Path(__file__).resolve().parents[2] /
'packages/kumo-engine/test/desktop_reload_qualification.py')
SPEC = importlib.util.spec_from_file_location('desktop_reload', SCRIPT)
RELOAD = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(RELOAD)


def test_worker_environment(tmp_path, monkeypatch):
for name in ('CI', 'GITHUB_ACTIONS', 'PYTEST_CURRENT_TEST',
'SYNC_DATA_LOADER', 'KUMO_CACHE', 'DISKGRAPH_BINARY'):
monkeypatch.setenv(name, 'masked')
monkeypatch.setenv('CUDA_VISIBLE_DEVICES', '0')
env = RELOAD.worker_environment(tmp_path, 'cpu', {'DRQ_HIDE_FAISS': '1'})
assert not any(value == 'masked' for value in env.values())
for name in ('HOME', 'USERPROFILE'):
assert env[name] == str(tmp_path / 'home')
for name in ('TMPDIR', 'TEMP', 'TMP'):
assert env[name] == str(tmp_path / 'tmp')
assert env['CUDA_VISIBLE_DEVICES'] == ''
assert env['DRQ_HIDE_FAISS'] == '1'


def test_native_windows_label(monkeypatch):
monkeypatch.setattr(sys, 'platform', 'win32')
assert RELOAD.label('cpu').startswith('windows-native-cpu-py')


def test_windows_timeout_terminates_tree(monkeypatch):
process = Mock(pid=123)
process.poll.return_value = None
run = Mock()
monkeypatch.setattr(sys, 'platform', 'win32')
monkeypatch.setattr(subprocess, 'run', run)
RELOAD.finish_process(process)
run.assert_called_once_with(
['taskkill', '/PID', '123', '/T', '/F'],
check=True, capture_output=True)
process.wait.assert_called_once_with(timeout=30)


def test_windows_completed_process_is_reaped(monkeypatch):
process = Mock()
process.poll.return_value = 0
run = Mock()
monkeypatch.setattr(sys, 'platform', 'win32')
monkeypatch.setattr(subprocess, 'run', run)
RELOAD.finish_process(process)
run.assert_not_called()
process.wait.assert_called_once_with(timeout=30)


@pytest.mark.parametrize('exited', [True, False])
def test_windows_taskkill_exit_race(monkeypatch, exited):
process = Mock(pid=123)
process.poll.side_effect = [None, 0 if exited else None]
run = Mock(side_effect=subprocess.CalledProcessError(128, 'taskkill'))
monkeypatch.setattr(sys, 'platform', 'win32')
monkeypatch.setattr(subprocess, 'run', run)
if exited:
RELOAD.finish_process(process)
process.wait.assert_called_once_with(timeout=30)
else:
with pytest.raises(subprocess.CalledProcessError):
RELOAD.finish_process(process)


@pytest.mark.skipif(sys.platform == 'win32', reason='POSIX process group check')
def test_spawn_isolates_then_removes_storage(tmp_path, monkeypatch):
child = tmp_path / 'child.py'
fsutil.write_text(str(child), '''import json, os, sys
from pathlib import Path
out, base = Path(sys.argv[6]), Path(sys.argv[7])
out.with_suffix('.json').write_text(json.dumps({
'pid': os.getpid(), 'base': str(base),
'masked': [key for key in ('GITHUB_ACTIONS', 'SYNC_DATA_LOADER',
'PYTEST_CURRENT_TEST') if key in os.environ],
'temp': os.environ['TEMP'], 'home': os.environ['USERPROFILE']}))
''')
monkeypatch.setattr(RELOAD, '__file__', str(child))
monkeypatch.setenv('GITHUB_ACTIONS', 'true')
monkeypatch.setenv('SYNC_DATA_LOADER', '1')
out = tmp_path / 'produce'
assert RELOAD.spawn('produce', 'binary', 'cpu', tmp_path, out) == 0
record = json.loads(fsutil.read_text(str(out.with_suffix('.json'))))
assert record['pid'] != os.getpid()
assert record['masked'] == []
assert record['temp'] == str(Path(record['base']) / 'tmp')
assert record['home'] == str(Path(record['base']) / 'home')
assert not fsutil.exists(record['base'])
34 changes: 34 additions & 0 deletions test/scripts/test_engine_wheel_workflow.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,3 +299,37 @@ def test_cpu_qualification_uses_candidate_artifacts_on_all_pythons():
# artifact while publishing the current build.
assert 'run-id' not in download['with']
assert 'github-token' not in download['with']


def test_windows_reload_blocks_publication_with_current_wheel():
caller = JOBS['platform-qualification']
assert caller['permissions']['id-token'] == 'write'
workflow = yaml.safe_load((WORKFLOW.parent /
'kumo-engine-platform-smoke.yml').read_text())
job = workflow['jobs']['windows-reload']
assert job['runs-on'] == 'windows-2022'
assert set(job['strategy']['matrix']['python']) == {
'3.11', '3.12', '3.13', '3.14'}
assert job['environment']['name'] == 'gha-kumo-workflow-tests'
download = next(step['with'] for step in job['steps']
if step.get('uses', '').startswith(
'actions/download-artifact@'))
assert download == {
'name': 'kumo-engine-wheel-windows-x86_64-${{ github.sha }}',
'path': '${{ runner.temp }}/windows-reload/artifacts'}
run = next(step['run'] for step in job['steps']
if step.get('name') ==
'Qualify the installed Windows wheel in fresh processes')
assert 'venv/Scripts/python.exe' in run
assert '--torch-backend=cpu "${wheels[0]}[cpu]"' in run
assert 'uv pip check --python "$python"' in run
assert 'desktop_reload_qualification.py run' in run
assert '--commit "$CANDIDATE_SHA"' in run
assert 'platform-qualification' in JOBS['attach-to-release']['needs']
build = _step('build', 'Test Windows path validation')['run']
assert '-p diskgraph-index --lib' in build
assert ('graph::scratch_cleanup_tests::'
'sync_directory_flushes_completed_entry') in build
assert '-- --exact | tee directory-sync-tests.log' in build
assert "grep -Eq '^test result: ok\\. 1 passed;'" in build
assert 'snapshot_mirror::tests | tee snapshot-mirror-tests.log' in build
Loading