From 71da0ed04eddce27ab6f474e6faf157f4e49fab7 Mon Sep 17 00:00:00 2001 From: danielmawhirter Date: Wed, 18 Nov 2026 16:17:54 -0600 Subject: [PATCH] Qualify native Windows CPU wheels in fresh processes --- .../workflows/kumo-engine-platform-smoke.yml | 79 +++++++++++++ .github/workflows/kumo-engine-wheel.yml | 14 +++ docs/development/releasing-kumo-engine.md | 9 ++ .../test/desktop_reload_qualification.py | 67 ++++++++--- .../test_desktop_reload_qualification.py | 105 ++++++++++++++++++ test/scripts/test_engine_wheel_workflow.py | 34 ++++++ 6 files changed, 290 insertions(+), 18 deletions(-) create mode 100644 test/scripts/test_desktop_reload_qualification.py diff --git a/.github/workflows/kumo-engine-platform-smoke.yml b/.github/workflows/kumo-engine-platform-smoke.yml index c8a5bf46f..628ff02bc 100644 --- a/.github/workflows/kumo-engine-platform-smoke.yml +++ b/.github/workflows/kumo-engine-platform-smoke.yml @@ -11,6 +11,85 @@ permissions: contents: read jobs: + windows-reload: + name: Native Windows CPU reload (Python ${{ matrix.python }}) + runs-on: windows-2022 + timeout-minutes: 60 + environment: {name: gha-kumo-workflow-tests, deployment: false} + permissions: {contents: read, id-token: write} + strategy: + fail-fast: false + matrix: + python: ['3.11', '3.12', '3.13', '3.14'] + defaults: {run: {shell: bash}} + env: + PYTHONPATH: '' + UV_LINK_MODE: copy + UV_NO_CONFIG: 'true' + CANDIDATE_SHA: ${{ github.sha }} + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: {ref: '${{ github.sha }}', persist-credentials: false} + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: kumo-engine-wheel-windows-x86_64-${{ github.sha }} + path: ${{ runner.temp }}/windows-reload/artifacts + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 + with: {enable-cache: false} + - id: aws + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: arn:aws:iam::926922431314:role/gha-kumo-workflow-tests + aws-region: us-west-2 + output-env-credentials: false + output-credentials: true + unset-current-credentials: true + - name: Download and check the pinned PR172 inputs + env: + AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }} + run: | + uv run --no-project --python 3.12 --with 'boto3>=1.34,<2' python -I - <<'PY' + import boto3, hashlib, json, os, tarfile + from pathlib import Path + spec = json.loads(Path('packages/kumo-engine/test/pr172/inputs.json').read_text()) + archive = Path(os.environ['RUNNER_TEMP'], 'reload-inputs.tar.gz') + boto3.client('s3', region_name=spec['region']).download_file(spec['bucket'], spec['key'], str(archive)) + if hashlib.sha256(archive.read_bytes()).hexdigest() != spec['sha256']: + raise SystemExit('PR172 input checksum differs') + with tarfile.open(archive) as source: + source.extractall(Path(os.environ['RUNNER_TEMP'], 'reload-inputs'), filter='data') + PY + - name: Qualify the installed Windows wheel in fresh processes + run: | + set -euo pipefail + [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ] || exec env -u ACTIONS_ID_TOKEN_REQUEST_URL -u ACTIONS_ID_TOKEN_REQUEST_TOKEN bash --noprofile --norc -eo pipefail "$0" + stage="$(cygpath -u "$RUNNER_TEMP")/windows-reload" + inputs="$(cygpath -u "$RUNNER_TEMP")/reload-inputs" + mkdir -p "$stage/test" "$stage/results" + cp packages/kumo-engine/test/{desktop_reload_qualification,wheel_gpu_smoke}.py "$stage/test/" + cd "$stage" + wheels=(artifacts/kumo_engine-*.whl) + [ "${#wheels[@]}" -eq 1 ] && [ -f "${wheels[0]}" ] + uv venv --python '${{ matrix.python }}' venv + python="$stage/venv/Scripts/python.exe" + uv pip install --python "$python" --torch-backend=cpu "${wheels[0]}[cpu]" 2>&1 | tee results/install.log + uv pip check --python "$python" 2>&1 | tee results/dependencies.log + uv pip freeze --python "$python" > results/freeze.txt + "$python" -I test/desktop_reload_qualification.py run --wheel "${wheels[0]}" --device cpu \ + --data-dir "$inputs" --output-dir results --commit "$CANDIDATE_SHA" + - if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: engine-wheel-reload-windows-native-cpu-py${{ matrix.python }}-${{ github.sha }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/windows-reload/results + !${{ runner.temp }}/windows-reload/results/**/*.parquet + !${{ runner.temp }}/windows-reload/results/*/*.json + if-no-files-found: warn + retention-days: 14 + smoke: name: CPU wheel (${{ matrix.target.platform }}, Python ${{ matrix.python }}) timeout-minutes: 60 diff --git a/.github/workflows/kumo-engine-wheel.yml b/.github/workflows/kumo-engine-wheel.yml index 699b02bef..945868f66 100644 --- a/.github/workflows/kumo-engine-wheel.yml +++ b/.github/workflows/kumo-engine-wheel.yml @@ -178,6 +178,17 @@ jobs: test --manifest-path packages/diskgraph/Cargo.toml \ --release --locked -p diskgraph-ingest --lib \ rejects_windows_profiling_before_output_changes + cargo --config packages/diskgraph/.cargo/config.toml \ + test --manifest-path packages/diskgraph/Cargo.toml \ + --release --locked -p diskgraph-index --lib \ + graph::scratch_cleanup_tests::sync_directory_flushes_completed_entry \ + -- --exact | tee directory-sync-tests.log + grep -Eq '^test result: ok\. 1 passed;' directory-sync-tests.log + cargo --config packages/diskgraph/.cargo/config.toml \ + test --manifest-path packages/diskgraph/Cargo.toml \ + --release --locked -p diskgraph-ingest --lib \ + snapshot_mirror::tests | tee snapshot-mirror-tests.log + grep -Eq '^test result: ok\. [1-9][0-9]* passed;' snapshot-mirror-tests.log # GitHub-hosted artifact storage, not S3: no AWS role to provision. # Native Windows wheels are experimental internal CI artifacts only. @@ -270,6 +281,9 @@ jobs: platform-qualification: needs: [build] + permissions: + contents: read + id-token: write uses: ./.github/workflows/kumo-engine-platform-smoke.yml attach-to-release: diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index a50f7676a..0e7282afa 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -52,6 +52,15 @@ targets does not establish published availability or installed-wheel runtime qualification. Native Windows builds and tests are experimental: their wheels remain CI artifacts and are excluded from public release attachments and Kitmaker publication. The supported Windows route for this release is WSL2. +Every wheel build also qualifies the native Windows CPU wheel on Python +3.11–3.14 in fresh producer and consumer processes, with the `cpu` extra for +Faiss retrieval. This check is a prerequisite for release attachment; it does +not enable Windows publication. It exports and restores both models and +snapshots after deleting the producer's temporary storage, and checks ordinary +loader defaults without CI or synchronous-loader overrides. Results are saved +in `engine-wheel-reload-windows-native-cpu-*` artifacts. Native Windows CUDA +is not qualified by this check. The GitHub-hosted Windows account may have +administrator privileges; this is not restricted-account qualification. The merger reads every bundled wheel's `Requires-Python` and writes their intersection into the final wheel. diff --git a/packages/kumo-engine/test/desktop_reload_qualification.py b/packages/kumo-engine/test/desktop_reload_qualification.py index d38157102..a947ec606 100644 --- a/packages/kumo-engine/test/desktop_reload_qualification.py +++ b/packages/kumo-engine/test/desktop_reload_qualification.py @@ -14,7 +14,6 @@ import platform import random import re -import resource import shutil import signal import subprocess @@ -29,6 +28,9 @@ from urllib.parse import urlsplit from urllib.request import url2pathname +if sys.platform != 'win32': + import resource + GPU_SMOKE = Path(__file__).with_name('wheel_gpu_smoke.py') SPEC = util.spec_from_file_location(GPU_SMOKE.stem, GPU_SMOKE) tasks = util.module_from_spec(SPEC) @@ -53,6 +55,8 @@ def label(device): """Name the OS or macOS version, CPU, device and Python of this run.""" if sys.platform == 'darwin': host = 'macos-' + platform.mac_ver()[0].split('.')[0] + elif sys.platform == 'win32': + host = 'windows-native' elif 'microsoft' in platform.release().lower(): host = 'windows-wsl' else: @@ -187,30 +191,56 @@ def worker(phase, task, device, durable, out, base, data): out.with_suffix('.json').write_text(json.dumps(meta, default=sorted)) +def worker_environment(base, device, extra): + """Give each process fresh caches and ordinary-user loader defaults.""" + drop = ('KUMO_', 'DISKGRAPH_') + masked = {'PYTEST_CURRENT_TEST', 'GITHUB_ACTIONS', 'SYNC_DATA_LOADER', 'CI'} + env = {k: v for k, v in os.environ.items() + if not k.startswith(drop) and k not in masked} + env.update(extra, HOME=str(base / 'home'), USERPROFILE=str(base / 'home'), + TMPDIR=str(base / 'tmp'), TEMP=str(base / 'tmp'), + TMP=str(base / 'tmp'), XDG_CACHE_HOME=str(base / 'cache'), + PYTORCH_ENABLE_MPS_FALLBACK='0') + if device == 'cpu': + env['CUDA_VISIBLE_DEVICES'] = '' + return env + + +def finish_process(process): + """Terminate timed-out workers and wait before deleting their storage.""" + if sys.platform == 'win32': + if process.poll() is None: + try: + subprocess.run(['taskkill', '/PID', str(process.pid), '/T', '/F'], + check=True, capture_output=True) + except subprocess.CalledProcessError: + if process.poll() is None: + raise + else: + with suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=30) + + def spawn(phase, task, device, durable, out, data=None, **extra): - """Run one worker in a new process. Then kill its process group and - delete its scratch, HOME, TMPDIR, XDG cache, cwd and fixture copy.""" - base = Path(tempfile.mkdtemp(prefix='drq-', dir='/tmp')) + """Run and reap a worker, terminating its process tree on timeout. + + Delete its scratch, HOME, TMPDIR, XDG cache, cwd and fixture copy. + """ + base = Path(tempfile.mkdtemp(prefix='drq-')) for name in ('home', 'cache', 'cwd', 'tmp', 'scratch'): (base / name).mkdir() - drop = ('KUMO_', 'DISKGRAPH_') # Could point at other binaries or data. - env = {k: v for k, v in os.environ.items() if not k.startswith(drop)} - env.update(extra, HOME=str(base / 'home'), TMPDIR=str(base / 'tmp'), - XDG_CACHE_HOME=str(base / 'cache'), - PYTORCH_ENABLE_MPS_FALLBACK='0') # No silent CPU fallback. - if device == 'cpu': # A CPU row must not reach a GPU. - env['CUDA_VISIBLE_DEVICES'] = '' + env = worker_environment(base, device, extra) command = [sys.executable, '-Iu', __file__, 'worker', phase, task, device] command += map(str, (durable, out, base, data)) with open(out.with_suffix('.log'), 'wb') as log: process = subprocess.Popen(command, cwd=base / 'cwd', env=env, stdout=log, stderr=subprocess.STDOUT, - start_new_session=True) + start_new_session=sys.platform != 'win32') try: return process.wait(timeout=3600) finally: - with suppress(ProcessLookupError): - os.killpg(process.pid, signal.SIGKILL) + finish_process(process) shutil.rmtree(base) @@ -296,7 +326,7 @@ def main(): summary = dict(label=label(args.device), commit=args.commit, started=now, wheel=args.wheel.name, wheel_sha256=None, cpus=os.cpu_count(), tasks={}) - work = Path(tempfile.mkdtemp(prefix='drq-work-', dir='/tmp')) + work = Path(tempfile.mkdtemp(prefix='drq-work-')) try: for task in TASKS: started = time.monotonic() @@ -313,9 +343,10 @@ def main(): error=f'{type(error).__name__}: {error}') result['seconds'] = round(time.monotonic() - started, 1) # The largest worker's ru_maxrss: KiB on Linux, bytes on macOS. - rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss - summary['peak_rss_mib'] = round( - rss / (1 << 20 if sys.platform == 'darwin' else 1 << 10)) + if sys.platform != 'win32': + rss = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + summary['peak_rss_mib'] = round( + rss / (1 << 20 if sys.platform == 'darwin' else 1 << 10)) print(task, json.dumps(result), flush=True) (output / 'summary.json').write_text(json.dumps(summary, indent=1)) finally: diff --git a/test/scripts/test_desktop_reload_qualification.py b/test/scripts/test_desktop_reload_qualification.py new file mode 100644 index 000000000..5f21d6e9f --- /dev/null +++ b/test/scripts/test_desktop_reload_qualification.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import Mock + +import pytest + +from io_utils import fsutil + +SCRIPT = (Path(__file__).resolve().parents[2] / + 'packages/kumo-engine/test/desktop_reload_qualification.py') +SPEC = importlib.util.spec_from_file_location('desktop_reload', SCRIPT) +RELOAD = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(RELOAD) + + +def test_worker_environment(tmp_path, monkeypatch): + for name in ('CI', 'GITHUB_ACTIONS', 'PYTEST_CURRENT_TEST', + 'SYNC_DATA_LOADER', 'KUMO_CACHE', 'DISKGRAPH_BINARY'): + monkeypatch.setenv(name, 'masked') + monkeypatch.setenv('CUDA_VISIBLE_DEVICES', '0') + env = RELOAD.worker_environment(tmp_path, 'cpu', {'DRQ_HIDE_FAISS': '1'}) + assert not any(value == 'masked' for value in env.values()) + for name in ('HOME', 'USERPROFILE'): + assert env[name] == str(tmp_path / 'home') + for name in ('TMPDIR', 'TEMP', 'TMP'): + assert env[name] == str(tmp_path / 'tmp') + assert env['CUDA_VISIBLE_DEVICES'] == '' + assert env['DRQ_HIDE_FAISS'] == '1' + + +def test_native_windows_label(monkeypatch): + monkeypatch.setattr(sys, 'platform', 'win32') + assert RELOAD.label('cpu').startswith('windows-native-cpu-py') + + +def test_windows_timeout_terminates_tree(monkeypatch): + process = Mock(pid=123) + process.poll.return_value = None + run = Mock() + monkeypatch.setattr(sys, 'platform', 'win32') + monkeypatch.setattr(subprocess, 'run', run) + RELOAD.finish_process(process) + run.assert_called_once_with( + ['taskkill', '/PID', '123', '/T', '/F'], + check=True, capture_output=True) + process.wait.assert_called_once_with(timeout=30) + + +def test_windows_completed_process_is_reaped(monkeypatch): + process = Mock() + process.poll.return_value = 0 + run = Mock() + monkeypatch.setattr(sys, 'platform', 'win32') + monkeypatch.setattr(subprocess, 'run', run) + RELOAD.finish_process(process) + run.assert_not_called() + process.wait.assert_called_once_with(timeout=30) + + +@pytest.mark.parametrize('exited', [True, False]) +def test_windows_taskkill_exit_race(monkeypatch, exited): + process = Mock(pid=123) + process.poll.side_effect = [None, 0 if exited else None] + run = Mock(side_effect=subprocess.CalledProcessError(128, 'taskkill')) + monkeypatch.setattr(sys, 'platform', 'win32') + monkeypatch.setattr(subprocess, 'run', run) + if exited: + RELOAD.finish_process(process) + process.wait.assert_called_once_with(timeout=30) + else: + with pytest.raises(subprocess.CalledProcessError): + RELOAD.finish_process(process) + + +@pytest.mark.skipif(sys.platform == 'win32', reason='POSIX process group check') +def test_spawn_isolates_then_removes_storage(tmp_path, monkeypatch): + child = tmp_path / 'child.py' + fsutil.write_text(str(child), '''import json, os, sys +from pathlib import Path +out, base = Path(sys.argv[6]), Path(sys.argv[7]) +out.with_suffix('.json').write_text(json.dumps({ + 'pid': os.getpid(), 'base': str(base), + 'masked': [key for key in ('GITHUB_ACTIONS', 'SYNC_DATA_LOADER', + 'PYTEST_CURRENT_TEST') if key in os.environ], + 'temp': os.environ['TEMP'], 'home': os.environ['USERPROFILE']})) +''') + monkeypatch.setattr(RELOAD, '__file__', str(child)) + monkeypatch.setenv('GITHUB_ACTIONS', 'true') + monkeypatch.setenv('SYNC_DATA_LOADER', '1') + out = tmp_path / 'produce' + assert RELOAD.spawn('produce', 'binary', 'cpu', tmp_path, out) == 0 + record = json.loads(fsutil.read_text(str(out.with_suffix('.json')))) + assert record['pid'] != os.getpid() + assert record['masked'] == [] + assert record['temp'] == str(Path(record['base']) / 'tmp') + assert record['home'] == str(Path(record['base']) / 'home') + assert not fsutil.exists(record['base']) diff --git a/test/scripts/test_engine_wheel_workflow.py b/test/scripts/test_engine_wheel_workflow.py index 7118df213..1cc73b5ed 100644 --- a/test/scripts/test_engine_wheel_workflow.py +++ b/test/scripts/test_engine_wheel_workflow.py @@ -299,3 +299,37 @@ def test_cpu_qualification_uses_candidate_artifacts_on_all_pythons(): # artifact while publishing the current build. assert 'run-id' not in download['with'] assert 'github-token' not in download['with'] + + +def test_windows_reload_blocks_publication_with_current_wheel(): + caller = JOBS['platform-qualification'] + assert caller['permissions']['id-token'] == 'write' + workflow = yaml.safe_load((WORKFLOW.parent / + 'kumo-engine-platform-smoke.yml').read_text()) + job = workflow['jobs']['windows-reload'] + assert job['runs-on'] == 'windows-2022' + assert set(job['strategy']['matrix']['python']) == { + '3.11', '3.12', '3.13', '3.14'} + assert job['environment']['name'] == 'gha-kumo-workflow-tests' + download = next(step['with'] for step in job['steps'] + if step.get('uses', '').startswith( + 'actions/download-artifact@')) + assert download == { + 'name': 'kumo-engine-wheel-windows-x86_64-${{ github.sha }}', + 'path': '${{ runner.temp }}/windows-reload/artifacts'} + run = next(step['run'] for step in job['steps'] + if step.get('name') == + 'Qualify the installed Windows wheel in fresh processes') + assert 'venv/Scripts/python.exe' in run + assert '--torch-backend=cpu "${wheels[0]}[cpu]"' in run + assert 'uv pip check --python "$python"' in run + assert 'desktop_reload_qualification.py run' in run + assert '--commit "$CANDIDATE_SHA"' in run + assert 'platform-qualification' in JOBS['attach-to-release']['needs'] + build = _step('build', 'Test Windows path validation')['run'] + assert '-p diskgraph-index --lib' in build + assert ('graph::scratch_cleanup_tests::' + 'sync_directory_flushes_completed_entry') in build + assert '-- --exact | tee directory-sync-tests.log' in build + assert "grep -Eq '^test result: ok\\. 1 passed;'" in build + assert 'snapshot_mirror::tests | tee snapshot-mirror-tests.log' in build