diff --git a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs index 4e90e9704..7348bb414 100644 --- a/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs +++ b/packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs @@ -597,7 +597,7 @@ pub fn snapshot_range_checksums( range_bytes: u64, ) -> std::result::Result>, CorruptionError> { use rayon::prelude::*; - use std::os::unix::fs::FileExt; + use std::io::{Read, Seek, SeekFrom}; const CHUNK: u64 = 1 << 20; if range_bytes == 0 { @@ -631,15 +631,18 @@ pub fn snapshot_range_checksums( path: path.clone(), detail: e.to_string(), }; - let file = std::fs::File::open(&path).map_err(io)?; + let mut file = std::fs::File::open(&path).map_err(io)?; let start = range * range_bytes; let end = entry.size_bytes.min(start + range_bytes); let mut hasher = crc32fast::Hasher::new(); let mut buffer = vec![0u8; CHUNK.min(end - start) as usize]; + if start < end { + file.seek(SeekFrom::Start(start)).map_err(io)?; + } let mut offset = start; while offset < end { let length = CHUNK.min(end - offset) as usize; - file.read_exact_at(&mut buffer[..length], offset).map_err(io)?; + file.read_exact(&mut buffer[..length]).map_err(io)?; hasher.update(&buffer[..length]); offset += length as u64; } @@ -1033,4 +1036,113 @@ mod tests { CorruptionError::Io { .. } )); } + + fn crc32_reference(bytes: &[u8]) -> u32 { + let mut crc = !0u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = (crc >> 1) ^ if crc & 1 == 1 { 0xedb8_8320 } else { 0 }; + } + } + !crc + } + + #[test] + fn checksums_cover_parallel_ranges() { + let tmp = tempfile::tempdir().unwrap(); + let span: usize = (1 << 20) + 37; + let first: Vec = (0..(span * 3 + 51)) + .map(|i| (i.wrapping_mul(73) ^ (i >> 7) ^ 17) as u8) + .collect(); + let second: Vec = (0..(span * 2 + 173)) + .map(|i| (i.wrapping_mul(19) ^ (i >> 5) ^ 211) as u8) + .collect(); + write_marker_files( + tmp.path(), + &[("z.bin", &first), ("a.bin", &second), ("empty", b"")], + ); + let manifest = serde_json::to_vec(&Manifest { + schema_version: MANIFEST_SCHEMA_VERSION, + files: vec![ + ManifestEntry { + rel_path: "z.bin".into(), + size_bytes: first.len() as u64, + }, + ManifestEntry { + rel_path: "empty".into(), + size_bytes: 0, + }, + ManifestEntry { + rel_path: "a.bin".into(), + size_bytes: second.len() as u64, + }, + ], + }) + .unwrap(); + std::fs::write(tmp.path().join(MANIFEST_FILENAME), &manifest).unwrap(); + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(4) + .build() + .unwrap(); + let actual = pool + .install(|| snapshot_range_checksums(tmp.path(), span as u64)) + .unwrap(); + assert_eq!(actual.len(), 4); + for (name, bytes) in [ + ("z.bin", first.as_slice()), + ("a.bin", second.as_slice()), + (MANIFEST_FILENAME, manifest.as_slice()), + ] { + assert_eq!( + actual[name], + bytes.chunks(span).map(crc32_reference).collect::>(), + "checksum order differs for {name}" + ); + } + assert_eq!(actual["empty"], vec![0]); + } + + #[test] + fn checksums_reject_invalid_inputs() { + let tmp = tempfile::tempdir().unwrap(); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 0).unwrap_err(), + CorruptionError::Io { path, detail } + if path == tmp.path() && detail == "range_bytes must be positive" + )); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 16).unwrap_err(), + CorruptionError::MissingManifest(path) + if path == tmp.path().join(MANIFEST_FILENAME) + )); + write_manifest( + tmp.path(), + vec![ManifestEntry { + rel_path: "missing.bin".into(), + size_bytes: 0, + }], + ); + assert!(matches!( + snapshot_range_checksums(tmp.path(), 16).unwrap_err(), + CorruptionError::Io { path, .. } if path == tmp.path().join("missing.bin") + )); + } + + #[test] + fn checksums_reject_short_files() { + let tmp = tempfile::tempdir().unwrap(); + write_marker_files(tmp.path(), &[("short.bin", b"short")]); + write_manifest( + tmp.path(), + vec![ManifestEntry { + rel_path: "short.bin".into(), + size_bytes: (1 << 20) + 91, + }], + ); + assert!(matches!( + snapshot_range_checksums(tmp.path(), (1 << 20) + 37).unwrap_err(), + CorruptionError::Io { path, .. } if path == tmp.path().join("short.bin") + )); + } } diff --git a/packages/kumo-core/src/kumo_core/training/config.py b/packages/kumo-core/src/kumo_core/training/config.py index cfd7e9b73..62fe236ad 100644 --- a/packages/kumo-core/src/kumo_core/training/config.py +++ b/packages/kumo-core/src/kumo_core/training/config.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import sys import warnings from dataclasses import field from enum import Enum @@ -248,6 +249,8 @@ def _coerce_num_neighbors(cls, data: Any) -> Any: def _dev_shm_bytes() -> int | None: + if sys.platform != 'linux': + return None try: stat = os.statvfs('/dev/shm') except OSError: diff --git a/packages/kumo-core/tests/test_loader_config.py b/packages/kumo-core/tests/test_loader_config.py new file mode 100644 index 000000000..aa6a1de1b --- /dev/null +++ b/packages/kumo-core/tests/test_loader_config.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +import pytest +import torch +from kumo_core.training import config + + +@pytest.mark.parametrize('platform', ['win32', 'darwin']) +def test_default_workers_without_statvfs(monkeypatch, platform): + for name in ('PYTEST_CURRENT_TEST', 'GITHUB_ACTIONS', 'SYNC_DATA_LOADER'): + monkeypatch.delenv(name, raising=False) + monkeypatch.setattr(config.sys, 'platform', platform) + monkeypatch.delattr(config.os, 'statvfs', raising=False) + monkeypatch.setattr(torch.multiprocessing, 'cpu_count', lambda: 8) + + assert config.LoaderKwargs(batch_size=1024).num_workers == 8 + + +@pytest.mark.parametrize('capacity, expected', [(0, 0), (1, 0), (3, 3), (8, 8)]) +def test_linux_workers_respect_shared_memory(monkeypatch, capacity, expected): + monkeypatch.setattr(config.sys, 'platform', 'linux') + monkeypatch.setattr( + config.os, 'statvfs', + lambda path: SimpleNamespace(f_blocks=capacity, f_frsize=256 * 1024**2), + raising=False) + + if expected < 8: + with pytest.warns(RuntimeWarning, match='reducing DataLoader'): + assert config.shm_safe_num_workers(8) == expected + else: + assert config.shm_safe_num_workers(8) == expected + + +def test_missing_linux_shared_memory_preserves_workers(monkeypatch): + monkeypatch.setattr(config.sys, 'platform', 'linux') + + def unavailable(path): + raise FileNotFoundError(path) + + monkeypatch.setattr(config.os, 'statvfs', unavailable, raising=False) + assert config.shm_safe_num_workers(8) == 8 diff --git a/packages/kumo-engine/kumo/engine/_snapshot_io.py b/packages/kumo-engine/kumo/engine/_snapshot_io.py index f4765fcf1..35eac1d8c 100644 --- a/packages/kumo-engine/kumo/engine/_snapshot_io.py +++ b/packages/kumo-engine/kumo/engine/_snapshot_io.py @@ -7,6 +7,7 @@ import logging import os import re +import shutil import threading import time import uuid @@ -402,20 +403,23 @@ def holds_export(index: Path, manifest: dict) -> bool: def _download(root: str, staging: Path, client: FileSystem, item: Tuple[str, int, int], stop: threading.Event) -> None: + from kumo_core.io.filesystem import open_file + name, offset, length = item - with client.open_input_file(f'{root}/index/{name}') as reader: - fd = os.open(staging / name, os.O_WRONLY) - try: - for start in range(offset, offset + length, _CHUNK): - _stopped(stop) - want = min(_CHUNK, offset + length - start) - chunk = reader.read_at(want, start) - if len(chunk) != want: - raise ValueError(f'{name} is truncated') - os.pwrite(fd, chunk, start) - getattr(os, 'fdatasync', os.fsync)(fd) - finally: - os.close(fd) + with (client.open_input_file(f'{root}/index/{name}') as reader, + open_file(str(staging / name), 'r+b') as writer): + # Each range has its own stream, so parallel seeks cannot change + # another range's position. Binary writes preserve Windows bytes. + writer.seek(offset) + for start in range(offset, offset + length, _CHUNK): + _stopped(stop) + want = min(_CHUNK, offset + length - start) + chunk = reader.read_at(want, start) + if len(chunk) != want: + raise ValueError(f'{name} is truncated') + writer.write(chunk) + writer.flush() + getattr(os, 'fdatasync', os.fsync)(writer.fileno()) def _index_files( @@ -488,8 +492,7 @@ def load_into(connector: Any, path: str, root: str, manifest: dict, def _allocate(staging: Path, files: List[Tuple[str, int]]) -> None: staging.mkdir(parents=True) needed = sum(size for _, size in files) - disk = os.statvfs(staging) - if disk.f_bavail * disk.f_frsize < needed: + if shutil.disk_usage(staging).free < needed: raise OSError(f'Loading this snapshot needs {needed} free bytes ' f'in {staging}.') for name, size in files: diff --git a/packages/kumo-engine/test/test_snapshot_export.py b/packages/kumo-engine/test/test_snapshot_export.py index b4acf21fb..97a805f53 100644 --- a/packages/kumo-engine/test/test_snapshot_export.py +++ b/packages/kumo-engine/test/test_snapshot_export.py @@ -10,6 +10,7 @@ import threading import warnings import zlib +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import replace from functools import partial @@ -90,11 +91,16 @@ def _load(exports, location, **kwargs): return GraphSnapshot.load_from_connector(exports, path=location, **kwargs) -@pytest.mark.parametrize('fallocate', [True, False]) +@pytest.mark.parametrize('portable', [False, True]) def test_export_load_round_trip_relocates_and_is_reused( - tmp_path, scratch, graph, exports, monkeypatch, fallocate): - if not fallocate: # As on macOS, which has no posix_fallocate. - monkeypatch.delattr(os, 'posix_fallocate', raising=False) + tmp_path, scratch, graph, exports, monkeypatch, portable): + if portable: + # Linux's disk_usage uses statvfs internally; Windows uses its native + # disk-space API. Keep that stdlib boundary available in this simulation. + disk = shutil.disk_usage(tmp_path) + monkeypatch.setattr(snapshot_io.shutil, 'disk_usage', lambda path: disk) + for name in ('statvfs', 'pwrite', 'posix_fallocate'): + monkeypatch.delattr(os, name, raising=False) monkeypatch.setattr(snapshot_io, '_CHUNK', 1024) monkeypatch.setattr(snapshot_io, '_RANGE_BYTES', 4096) snapshot = graph.snapshot() @@ -438,14 +444,57 @@ def test_load_refuses_before_downloading_when_disk_is_too_small( tmp_path, scratch, graph, exports, monkeypatch): location, _ = _export(graph.snapshot(), exports, tmp_path) scratch('gpu') - monkeypatch.setattr(snapshot_io.os, 'statvfs', - lambda path: SimpleNamespace(f_frsize=1, f_bavail=0)) + monkeypatch.setattr(snapshot_io.shutil, 'disk_usage', + lambda path: SimpleNamespace(free=0)) monkeypatch.setattr(snapshot_io, '_download', Mock(side_effect=AssertionError('downloaded'))) with pytest.raises(OSError, match='free bytes'): _load(exports, location) +def test_parallel_ranges_preserve_binary_bytes(tmp_path, monkeypatch): + source = tmp_path / 'source/index' + source.mkdir(parents=True) + payload = bytes(range(256)) * 97 + b'\r\n\x1a\n' + (source / 'data').write_bytes(payload) + staging = tmp_path / 'staging' + staging.mkdir() + (staging / 'data').write_bytes(b'\x00' * len(payload)) + monkeypatch.delattr(os, 'pwrite', raising=False) + monkeypatch.setattr(snapshot_io, '_CHUNK', 257) + stop = threading.Event() + ranges = [('data', offset, min(1021, len(payload) - offset)) + for offset in range(0, len(payload), 1021)] + + def download(item): + snapshot_io._download(str(source.parent), staging, LocalFileSystem(), + item, stop) + + with ThreadPoolExecutor(max_workers=8) as workers: + list(workers.map(download, reversed(ranges))) + + assert (staging / 'data').read_bytes() == payload + + +@pytest.mark.parametrize('free', [0, 1024]) +def test_portable_allocation_checks_free_space(tmp_path, monkeypatch, free): + for name in ('statvfs', 'posix_fallocate'): + monkeypatch.delattr(os, name, raising=False) + monkeypatch.setattr(snapshot_io.shutil, 'disk_usage', + lambda path: SimpleNamespace(free=free)) + staging = tmp_path / 'staging' + files = [('nested/data', 1024), ('empty', 0)] + + if free < 1024: + with pytest.raises(OSError, match='1024 free bytes'): + snapshot_io._allocate(staging, files) + assert list(staging.iterdir()) == [] + else: + snapshot_io._allocate(staging, files) + assert (staging / 'nested/data').stat().st_size == 1024 + assert (staging / 'empty').stat().st_size == 0 + + @pytest.mark.parametrize('index, edit', [ (False, lambda manifest: manifest.update(data_version='../../victim')), (False, lambda manifest: manifest.pop('crc32')),