Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 115 additions & 3 deletions packages/diskgraph/diskgraph-ingest/src/snapshot_mirror.rs
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,7 @@ pub fn snapshot_range_checksums(
range_bytes: u64,
) -> std::result::Result<BTreeMap<String, Vec<u32>>, CorruptionError> {
use rayon::prelude::*;
use std::os::unix::fs::FileExt;
use std::io::{Read, Seek, SeekFrom};

const CHUNK: u64 = 1 << 20;
if range_bytes == 0 {
Expand Down Expand Up @@ -631,15 +631,18 @@ pub fn snapshot_range_checksums(
path: path.clone(),
detail: e.to_string(),
};
let file = std::fs::File::open(&path).map_err(io)?;
let mut file = std::fs::File::open(&path).map_err(io)?;
let start = range * range_bytes;
let end = entry.size_bytes.min(start + range_bytes);
let mut hasher = crc32fast::Hasher::new();
let mut buffer = vec![0u8; CHUNK.min(end - start) as usize];
if start < end {
file.seek(SeekFrom::Start(start)).map_err(io)?;
}
let mut offset = start;
while offset < end {
let length = CHUNK.min(end - offset) as usize;
file.read_exact_at(&mut buffer[..length], offset).map_err(io)?;
file.read_exact(&mut buffer[..length]).map_err(io)?;
hasher.update(&buffer[..length]);
offset += length as u64;
}
Expand Down Expand Up @@ -1033,4 +1036,113 @@ mod tests {
CorruptionError::Io { .. }
));
}

fn crc32_reference(bytes: &[u8]) -> u32 {
let mut crc = !0u32;
for &byte in bytes {
crc ^= u32::from(byte);
for _ in 0..8 {
crc = (crc >> 1) ^ if crc & 1 == 1 { 0xedb8_8320 } else { 0 };
}
}
!crc
}

#[test]
fn checksums_cover_parallel_ranges() {
let tmp = tempfile::tempdir().unwrap();
let span: usize = (1 << 20) + 37;
let first: Vec<u8> = (0..(span * 3 + 51))
.map(|i| (i.wrapping_mul(73) ^ (i >> 7) ^ 17) as u8)
.collect();
let second: Vec<u8> = (0..(span * 2 + 173))
.map(|i| (i.wrapping_mul(19) ^ (i >> 5) ^ 211) as u8)
.collect();
write_marker_files(
tmp.path(),
&[("z.bin", &first), ("a.bin", &second), ("empty", b"")],
);
let manifest = serde_json::to_vec(&Manifest {
schema_version: MANIFEST_SCHEMA_VERSION,
files: vec![
ManifestEntry {
rel_path: "z.bin".into(),
size_bytes: first.len() as u64,
},
ManifestEntry {
rel_path: "empty".into(),
size_bytes: 0,
},
ManifestEntry {
rel_path: "a.bin".into(),
size_bytes: second.len() as u64,
},
],
})
.unwrap();
std::fs::write(tmp.path().join(MANIFEST_FILENAME), &manifest).unwrap();
let pool = rayon::ThreadPoolBuilder::new()
.num_threads(4)
.build()
.unwrap();
let actual = pool
.install(|| snapshot_range_checksums(tmp.path(), span as u64))
.unwrap();
assert_eq!(actual.len(), 4);
for (name, bytes) in [
("z.bin", first.as_slice()),
("a.bin", second.as_slice()),
(MANIFEST_FILENAME, manifest.as_slice()),
] {
assert_eq!(
actual[name],
bytes.chunks(span).map(crc32_reference).collect::<Vec<_>>(),
"checksum order differs for {name}"
);
}
assert_eq!(actual["empty"], vec![0]);
}

#[test]
fn checksums_reject_invalid_inputs() {
let tmp = tempfile::tempdir().unwrap();
assert!(matches!(
snapshot_range_checksums(tmp.path(), 0).unwrap_err(),
CorruptionError::Io { path, detail }
if path == tmp.path() && detail == "range_bytes must be positive"
));
assert!(matches!(
snapshot_range_checksums(tmp.path(), 16).unwrap_err(),
CorruptionError::MissingManifest(path)
if path == tmp.path().join(MANIFEST_FILENAME)
));
write_manifest(
tmp.path(),
vec![ManifestEntry {
rel_path: "missing.bin".into(),
size_bytes: 0,
}],
);
assert!(matches!(
snapshot_range_checksums(tmp.path(), 16).unwrap_err(),
CorruptionError::Io { path, .. } if path == tmp.path().join("missing.bin")
));
}

#[test]
fn checksums_reject_short_files() {
let tmp = tempfile::tempdir().unwrap();
write_marker_files(tmp.path(), &[("short.bin", b"short")]);
write_manifest(
tmp.path(),
vec![ManifestEntry {
rel_path: "short.bin".into(),
size_bytes: (1 << 20) + 91,
}],
);
assert!(matches!(
snapshot_range_checksums(tmp.path(), (1 << 20) + 37).unwrap_err(),
CorruptionError::Io { path, .. } if path == tmp.path().join("short.bin")
));
}
}
3 changes: 3 additions & 0 deletions packages/kumo-core/src/kumo_core/training/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0

import os
import sys
import warnings
from dataclasses import field
from enum import Enum
Expand Down Expand Up @@ -248,6 +249,8 @@ def _coerce_num_neighbors(cls, data: Any) -> Any:


def _dev_shm_bytes() -> int | None:
if sys.platform != 'linux':
return None
try:
stat = os.statvfs('/dev/shm')
except OSError:
Expand Down
45 changes: 45 additions & 0 deletions packages/kumo-core/tests/test_loader_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# All rights reserved.
# SPDX-License-Identifier: Apache-2.0

from types import SimpleNamespace

import pytest
import torch
from kumo_core.training import config


@pytest.mark.parametrize('platform', ['win32', 'darwin'])
def test_default_workers_without_statvfs(monkeypatch, platform):
for name in ('PYTEST_CURRENT_TEST', 'GITHUB_ACTIONS', 'SYNC_DATA_LOADER'):
monkeypatch.delenv(name, raising=False)
monkeypatch.setattr(config.sys, 'platform', platform)
monkeypatch.delattr(config.os, 'statvfs', raising=False)
monkeypatch.setattr(torch.multiprocessing, 'cpu_count', lambda: 8)

assert config.LoaderKwargs(batch_size=1024).num_workers == 8


@pytest.mark.parametrize('capacity, expected', [(0, 0), (1, 0), (3, 3), (8, 8)])
def test_linux_workers_respect_shared_memory(monkeypatch, capacity, expected):
monkeypatch.setattr(config.sys, 'platform', 'linux')
monkeypatch.setattr(
config.os, 'statvfs',
lambda path: SimpleNamespace(f_blocks=capacity, f_frsize=256 * 1024**2),
raising=False)

if expected < 8:
with pytest.warns(RuntimeWarning, match='reducing DataLoader'):
assert config.shm_safe_num_workers(8) == expected
else:
assert config.shm_safe_num_workers(8) == expected


def test_missing_linux_shared_memory_preserves_workers(monkeypatch):
monkeypatch.setattr(config.sys, 'platform', 'linux')

def unavailable(path):
raise FileNotFoundError(path)

monkeypatch.setattr(config.os, 'statvfs', unavailable, raising=False)
assert config.shm_safe_num_workers(8) == 8
33 changes: 18 additions & 15 deletions packages/kumo-engine/kumo/engine/_snapshot_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import logging
import os
import re
import shutil
import threading
import time
import uuid
Expand Down Expand Up @@ -402,20 +403,23 @@ def holds_export(index: Path, manifest: dict) -> bool:

def _download(root: str, staging: Path, client: FileSystem,
item: Tuple[str, int, int], stop: threading.Event) -> None:
from kumo_core.io.filesystem import open_file

name, offset, length = item
with client.open_input_file(f'{root}/index/{name}') as reader:
fd = os.open(staging / name, os.O_WRONLY)
try:
for start in range(offset, offset + length, _CHUNK):
_stopped(stop)
want = min(_CHUNK, offset + length - start)
chunk = reader.read_at(want, start)
if len(chunk) != want:
raise ValueError(f'{name} is truncated')
os.pwrite(fd, chunk, start)
getattr(os, 'fdatasync', os.fsync)(fd)
finally:
os.close(fd)
with (client.open_input_file(f'{root}/index/{name}') as reader,
open_file(str(staging / name), 'r+b') as writer):
# Each range has its own stream, so parallel seeks cannot change
# another range's position. Binary writes preserve Windows bytes.
writer.seek(offset)
for start in range(offset, offset + length, _CHUNK):
_stopped(stop)
want = min(_CHUNK, offset + length - start)
chunk = reader.read_at(want, start)
if len(chunk) != want:
raise ValueError(f'{name} is truncated')
writer.write(chunk)
writer.flush()
getattr(os, 'fdatasync', os.fsync)(writer.fileno())


def _index_files(
Expand Down Expand Up @@ -488,8 +492,7 @@ def load_into(connector: Any, path: str, root: str, manifest: dict,
def _allocate(staging: Path, files: List[Tuple[str, int]]) -> None:
staging.mkdir(parents=True)
needed = sum(size for _, size in files)
disk = os.statvfs(staging)
if disk.f_bavail * disk.f_frsize < needed:
if shutil.disk_usage(staging).free < needed:
raise OSError(f'Loading this snapshot needs {needed} free bytes '
f'in {staging}.')
for name, size in files:
Expand Down
61 changes: 55 additions & 6 deletions packages/kumo-engine/test/test_snapshot_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import threading
import warnings
import zlib
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from dataclasses import replace
from functools import partial
Expand Down Expand Up @@ -90,11 +91,16 @@ def _load(exports, location, **kwargs):
return GraphSnapshot.load_from_connector(exports, path=location, **kwargs)


@pytest.mark.parametrize('fallocate', [True, False])
@pytest.mark.parametrize('portable', [False, True])
def test_export_load_round_trip_relocates_and_is_reused(
tmp_path, scratch, graph, exports, monkeypatch, fallocate):
if not fallocate: # As on macOS, which has no posix_fallocate.
monkeypatch.delattr(os, 'posix_fallocate', raising=False)
tmp_path, scratch, graph, exports, monkeypatch, portable):
if portable:
# Linux's disk_usage uses statvfs internally; Windows uses its native
# disk-space API. Keep that stdlib boundary available in this simulation.
disk = shutil.disk_usage(tmp_path)
monkeypatch.setattr(snapshot_io.shutil, 'disk_usage', lambda path: disk)
for name in ('statvfs', 'pwrite', 'posix_fallocate'):
monkeypatch.delattr(os, name, raising=False)
monkeypatch.setattr(snapshot_io, '_CHUNK', 1024)
monkeypatch.setattr(snapshot_io, '_RANGE_BYTES', 4096)
snapshot = graph.snapshot()
Expand Down Expand Up @@ -438,14 +444,57 @@ def test_load_refuses_before_downloading_when_disk_is_too_small(
tmp_path, scratch, graph, exports, monkeypatch):
location, _ = _export(graph.snapshot(), exports, tmp_path)
scratch('gpu')
monkeypatch.setattr(snapshot_io.os, 'statvfs',
lambda path: SimpleNamespace(f_frsize=1, f_bavail=0))
monkeypatch.setattr(snapshot_io.shutil, 'disk_usage',
lambda path: SimpleNamespace(free=0))
monkeypatch.setattr(snapshot_io, '_download',
Mock(side_effect=AssertionError('downloaded')))
with pytest.raises(OSError, match='free bytes'):
_load(exports, location)


def test_parallel_ranges_preserve_binary_bytes(tmp_path, monkeypatch):
source = tmp_path / 'source/index'
source.mkdir(parents=True)
payload = bytes(range(256)) * 97 + b'\r\n\x1a\n'
(source / 'data').write_bytes(payload)
staging = tmp_path / 'staging'
staging.mkdir()
(staging / 'data').write_bytes(b'\x00' * len(payload))
monkeypatch.delattr(os, 'pwrite', raising=False)
monkeypatch.setattr(snapshot_io, '_CHUNK', 257)
stop = threading.Event()
ranges = [('data', offset, min(1021, len(payload) - offset))
for offset in range(0, len(payload), 1021)]

def download(item):
snapshot_io._download(str(source.parent), staging, LocalFileSystem(),
item, stop)

with ThreadPoolExecutor(max_workers=8) as workers:
list(workers.map(download, reversed(ranges)))

assert (staging / 'data').read_bytes() == payload


@pytest.mark.parametrize('free', [0, 1024])
def test_portable_allocation_checks_free_space(tmp_path, monkeypatch, free):
for name in ('statvfs', 'posix_fallocate'):
monkeypatch.delattr(os, name, raising=False)
monkeypatch.setattr(snapshot_io.shutil, 'disk_usage',
lambda path: SimpleNamespace(free=free))
staging = tmp_path / 'staging'
files = [('nested/data', 1024), ('empty', 0)]

if free < 1024:
with pytest.raises(OSError, match='1024 free bytes'):
snapshot_io._allocate(staging, files)
assert list(staging.iterdir()) == []
else:
snapshot_io._allocate(staging, files)
assert (staging / 'nested/data').stat().st_size == 1024
assert (staging / 'empty').stat().st_size == 0


@pytest.mark.parametrize('index, edit', [
(False, lambda manifest: manifest.update(data_version='../../victim')),
(False, lambda manifest: manifest.pop('crc32')),
Expand Down
Loading