diff --git a/.github/workflows/kumo-pytorch-diskgraph.yml b/.github/workflows/kumo-pytorch-diskgraph.yml index e2d4bbcc2..3e312b62a 100644 --- a/.github/workflows/kumo-pytorch-diskgraph.yml +++ b/.github/workflows/kumo-pytorch-diskgraph.yml @@ -13,7 +13,13 @@ name: kumo-pytorch DiskGraph on: # yamllint disable-line rule:truthy pull_request: paths: + - 'pyproject.toml' - 'packages/experimental/kumo-pytorch/**' + - 'packages/kumo-engine/**' + - 'packages/kumo-core/**' + - 'packages/kumo-api/**' + - 'packages/kumo-pql/**' + - 'packages/kumo-settings/**' - 'packages/diskgraph/**' - 'scripts/diskgraph' - '.github/workflows/_diskgraph-wheel.yml' @@ -22,7 +28,13 @@ on: # yamllint disable-line rule:truthy push: branches: [main] paths: + - 'pyproject.toml' - 'packages/experimental/kumo-pytorch/**' + - 'packages/kumo-engine/**' + - 'packages/kumo-core/**' + - 'packages/kumo-api/**' + - 'packages/kumo-pql/**' + - 'packages/kumo-settings/**' - 'packages/diskgraph/**' - 'scripts/diskgraph' - '.github/workflows/_diskgraph-wheel.yml' @@ -150,3 +162,26 @@ jobs: run: | ../fresh/bin/python -m pytest -p no:cacheprovider -m notebook --require-imports -q \ --durations=0 tests/test_notebooks.py + - name: Build the engine and its workspace dependencies as wheels + working-directory: . + run: | + set -euo pipefail + for package in kumo-api kumo-pql kumo-settings kumo-core kumo-engine; do + uv build --wheel --package "$package" --out-dir "$RUNNER_TEMP/engine-wheels" + done + cp packages/kumo-engine/test/test_pytorch_snapshot.py "$RUNNER_TEMP/stage/tests/" + - name: Install the engine wheels alongside PyTorch and DiskGraph + working-directory: ${{ runner.temp }} + run: | + uv pip install --python fresh/bin/python --torch-backend cpu engine-wheels/*.whl + uv pip check --python fresh/bin/python + - name: Connector snapshot to columnar batches, from installed wheels + working-directory: ${{ runner.temp }}/stage + run: | + ../fresh/bin/python -I -c ' + import pathlib, sys + import diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings + for module in (diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings): + assert pathlib.Path(module.__file__).is_relative_to(sys.prefix), module.__file__ + ' + ../fresh/bin/python -m pytest -p no:cacheprovider -q --require-imports tests/test_pytorch_snapshot.py diff --git a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md index 48752326f..fa8f81034 100644 --- a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md +++ b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md @@ -179,8 +179,14 @@ read. The index holds rows, key maps and relations. It holds no anchors, labels or fitted encoders, so several contracts over the same tables can share one index. -Fitting keeps the column statistics of encoders over stored columns in -`col_stats/`, so it needs the index directory to be writable. Each encoder +By default, fitting keeps column statistics of encoders over stored columns in +the index's `col_stats/`, so that directory must be writable. For the ordinary +`DiskGraphEngine(index=...)` constructor, omitting `work_dir` or explicitly setting +it to the index path uses this same layout and reuses the same caches. Paths that +resolve to the index, including symlinks, are equivalent. Other paths inside the +index are rejected. External work directories and their writable roots must be +disjoint from the base, including after resolving symlinks. Supply an external +`work_dir` to put newly computed statistics and task state there instead. Each encoder cutoff has one directory there, with a `.col_stats.json` file per table. A later fit at the same cutoff computes only the columns and statistics those files lack, and adds them. Ingest computes the statistics for the cutoffs in @@ -195,6 +201,74 @@ it was built. With `ingest="never"`, `prepare` requires an index that already exists, which is what you usually want in a scoring job. +### Open an existing snapshot or index + +`DiskGraphEngine.from_snapshot(snapshot, work_dir=...)` consumes an existing +`kumo.engine.GraphSnapshot`. Use the engine's existing snapshot loader to +restore an exported snapshot first. `DiskGraphEngine.from_index(index_path, +work_dir=...)` opens the native index directory directly, including a +PyTorch-produced index. Neither factory exports sources or ingests the base. + + +```python +engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir="/scratch/loader") +contract = api.DataLoaderSpec( + schema=engine.schema, + sample=sample_spec, + batch=batch_spec, +) +with api.prepare(contract=contract, engine=engine) as plan: + with plan.materialize(anchors) as task: + for batch in task.batches(output_format="columnar"): + graph = batch.inputs["graph"] +``` + +`SampleSpec` and `BatchSpec` retain their usual meaning. `engine.schema` derives +stored types, keys, clocks and relationships from the existing native config +and metadata. PyTorch-produced indexes retain their logical relation aliases; +engine/native indexes initially use native FK names. A caller can provide a +compatible schema projection and relation aliases: binding checks endpoints, +FK columns and direction against the actual index. `prepare` rejects a Dataset +alongside an existing-index engine. Anchors still select entity/time requests; +neighborhoods are sampled during iteration. + +The base may be read-only. Task extensions, scalar-materialization scratch, +new statistics, tokenizers and token-length caches go under the external work +directory; existing immutable base caches can be read in place. Closing a task +cleans up its temporary files. Derived caches remain for reuse. No new base +metadata file is written. Fitted-state statistics provenance records absolute paths +in external mode (including reused base files); these are diagnostic references, +not inputs used when restoring fitted encoders. They can refer to the old location +after relocation. External statistics share one directory per artifact and cutoff; +the native writer locks, merges existing keys, and atomically replaces each file. +Concurrent requests may duplicate computation but do not discard published keys. +External storage requires a DiskGraph build supporting `cutoff_col_stats(output_dir=...)`; +preparation checks that capability because older and newer development builds can +share version 0.2.0. Install DiskGraph from the matching repository revision. +Existing snapshot identity or PyTorch producer fingerprints survive relocation. A bare completed native index without producer +identity is hashed in full on each opening, which can be expensive for large artifacts. + +Opening requires the resolved `diskgraph_config.json` and the producer's +completion metadata. Engine snapshots use the native inventory/integrity check; +PyTorch indexes use their existing manifest, published after ingestion. Older +artifacts missing resolved config are refused. Native edge-table layouts, +end-time validity and unsupported stored types (including lists) are refused +explicitly. Snapshot adoption does not recover relations omitted during ingestion +or enable additional sampling strategies. + +For **source ingestion/reuse** after connector restaging, pass +`DiskGraphEngine(index=..., work_dir=..., source_versions=...)`. Map every table +to `{"source": "connector/source-coordinate", "version": "connector-version"}`. +Equal trusted identities/versions and schema permit reuse across export paths; +a changed identity/version or schema refuses reuse. Without trustworthy versions, +keep the existing file-validation mode. This opt-in trusts the caller to change +the version whenever source data changes; reuse does not check Parquet contents, +file sizes or mtimes. Reusing a stale version token therefore reuses the old index. +Physical file checks still detect source changes during initial ingestion. +Opening an already materialized index +never checks whether its sources now have a newer version; refreshing sources +remains an explicit producer operation. + `output_format="model"` lets the DiskGraph engine build the batch's tensors column by column, without building a Python record per event. It refuses raw types that a model tensor can't hold (strings, dates, decimals) before fetching any diff --git a/packages/experimental/kumo-pytorch/docs/api/README.md b/packages/experimental/kumo-pytorch/docs/api/README.md index 9b54cb972..a9756b849 100644 --- a/packages/experimental/kumo-pytorch/docs/api/README.md +++ b/packages/experimental/kumo-pytorch/docs/api/README.md @@ -108,7 +108,7 @@ The [guide](../README.md) explains the concepts in order. The behavior spec, wit | [`InputExposureWarning`](targets.md#inputexposurewarning) | `kumo.pytorch.api` | A masked value is also selected by an unmasked named input. | | [`DataLoaderSpec`](lifecycle.md#dataloaderspec) | `kumo.pytorch.api` | Schema, sampled values, and explicitly encoded model outputs. | | [`check`](lifecycle.md#check) | `kumo.pytorch.api` | Return everything wrong with this combination, without raising. | -| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind the dataset through the engine. | +| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind sources or an existing index through the engine. | | [`preview`](lifecycle.md#preview) | `kumo.pytorch.api` | Build up to eight raw samples and one model-facing batch. | | [`Plan`](lifecycle.md#plan) | `kumo.pytorch.api` | A contract bound to a dataset through an engine, before any anchors. | | [`Task`](lifecycle.md#task) | `kumo.pytorch.api` | Anchors bound to a plan: what is iterated, served from, and written out. | diff --git a/packages/experimental/kumo-pytorch/docs/api/engines.md b/packages/experimental/kumo-pytorch/docs/api/engines.md index 60984b721..1ea55e042 100644 --- a/packages/experimental/kumo-pytorch/docs/api/engines.md +++ b/packages/experimental/kumo-pytorch/docs/api/engines.md @@ -299,6 +299,8 @@ class DiskGraphEngine( num_shards: int = 1, ingest: Literal['if_missing', 'never'] = 'if_missing', encoder_cutoffs: Sequence[str | int | datetime | date | None] = (), + work_dir: str | Path | None = None, + source_versions: Mapping[str, Mapping[str, str]] | None = None, ) ``` @@ -330,6 +332,54 @@ step, before any data is read. - `num_shards`: index shards. - `ingest`: `"if_missing"` builds the index when absent; `"never"` requires an existing one. A mismatching manifest is an error, never permission to reuse. - `encoder_cutoffs`: `encoder_cutoff` values whose column statistics an ingest computes. +- `work_dir`: task/cache directory. Omit it or pass the index path (including an alias resolving to it) for the original index-local layout and cache reuse. Other paths must be outside the index; existing-index factories require an external directory. +- `source_versions`: optional table-to-identity mapping for source ingestion/reuse. Each entry has nonempty `source` (connector plus source identity) and `version` strings. These trusted connector versions replace file path/size/mtime identity; missing versions are never treated as equal. Omit for legacy file validation. The caller must change the version whenever source data changes: reuse does not compare Parquet contents, file sizes or mtimes in this opt-in mode. + +#### `DiskGraphEngine.from_index` + +*classmethod* + +```python +DiskGraphEngine.from_index( + index: str | Path, + *, + work_dir: str | Path, + num_threads: int | None = None, +) -> DiskGraphEngine +``` + +Open a complete native index without its original sources or any base writes. + +Read schema/relations from existing metadata. Older indexes without the +resolved native config are refused. Native edge tables and non-scalar +stored columns are not supported. Bare indexes without producer identity +are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan. +`work_dir` holds task extensions and derived caches outside the base. + +#### `DiskGraphEngine.from_snapshot` + +*classmethod* + +```python +DiskGraphEngine.from_snapshot( + snapshot: Any, + *, + work_dir: str | Path, + num_threads: int | None = None, +) -> DiskGraphEngine +``` + +Bind a kumo.engine GraphSnapshot using its existing index and identity. + +Uses public `index_path` and `id` fields; importing kumo.engine is +unnecessary. Load exported snapshots through the existing engine loader. +No source export, base ingestion or additional metadata file is needed. + +#### `DiskGraphEngine.schema` + +*property* + +Resolved stored schema for a from_index/from_snapshot engine. #### `DiskGraphEngine.check_contract` diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md index 85aefeb1d..264eabe86 100644 --- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md +++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md @@ -144,8 +144,8 @@ A readable multi-line summary of the contract. ```python def check( contract: DataLoaderSpec, - dataset: Dataset, - engine: Engine, + dataset: Dataset | None = None, + engine: Engine | None = None, *, anchors: AnchorTable | None = None, inference: bool = False, @@ -169,14 +169,18 @@ The whole contract must still be well formed. ```python def prepare( contract: DataLoaderSpec, - dataset: Dataset, - engine: Engine, + dataset: Dataset | None = None, + engine: Engine | None = None, *, inference: bool = False, ) -> Plan ``` -Validate, then bind the dataset through the engine. +Validate, then bind sources or an existing index through the engine. + +Omit `dataset` only for a DiskGraphEngine created with `from_index` or +`from_snapshot`. Its stored schema supplies omitted types; it never reads +the original sources or rebuilds the base. `engine` is always required. Every check that does not need data happens first, so an unexecutable combination is refused before a row is read. Anchors, and everything fitted @@ -221,7 +225,7 @@ Called as `preview(task, requests)` or ```python class Plan( contract: DataLoaderSpec, - dataset: Dataset, + dataset: Dataset | None, engine: Engine, source: SampleSource, *, diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py new file mode 100644 index 000000000..b8c34a8a2 --- /dev/null +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Storage policy for index-local and external consumer state.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + + +def normalize_work_dir( + index: str | Path, work_dir: str | Path | None, *, external_only: bool = False +) -> Path | None: + """Use the legacy layout for an omitted or explicitly equal index path.""" + base = Path(index).resolve() + work = None if work_dir is None else Path(work_dir).resolve() + if not external_only and (work is None or work == base): + return None + if work is None: + raise ValueError("work_dir must be outside the base index") + # Validate every writable root before creating any of them, including symlinks. + for destination in (work, work / "tasks", work / "train_tables", work / "cache"): + resolved = destination.resolve() + if resolved.is_relative_to(base) or base.is_relative_to(resolved): + raise ValueError( + "work_dir and its writable roots must be outside and disjoint from the base index" + ) + return work + + +class ColumnCache: + """Read immutable base caches and publish missing entries to the writable root.""" + + def __init__(self, index: str | Path, root: str | Path | None = None): + self.index = Path(index) + self.root = self.index if root is None else Path(root) + self.external = self.root.resolve() != self.index.resolve() + if not self.external: + self.root = self.index + + @classmethod + def for_work_dir(cls, index: str | Path, work_dir: Path | None, identity: str): + root = ( + None + if work_dir is None + else work_dir / "cache" / hashlib.sha256(identity.encode()).hexdigest() + ) + return cls(index, root) + + def destination(self, name: str) -> Path: + return self.root / "col_stats" / name + + def candidates(self, name: str, *, base_name: str | None = None) -> tuple[Path, ...]: + base = self.index / "col_stats" / (name if base_name is None else base_name) + return tuple(dict.fromkeys((base, self.destination(name)))) + + def resolve(self, name: str) -> Path: + return next( + (path for path in self.candidates(name) if path.is_file()), self.destination(name) + ) + + def reference(self, path: Path) -> str: + return str(path if self.external else path.relative_to(self.index / "col_stats")) diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py index d1cf37703..e219759e7 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py @@ -20,6 +20,7 @@ import numpy as np +from kumo.pytorch._storage import ColumnCache from kumo.pytorch.contract.batch import ( Composite, EntitySource, @@ -243,7 +244,7 @@ def token_lengths_file( def read_token_lengths( - index: str | Path, entry: Mapping[str, Any] + index: str | Path, entry: Mapping[str, Any], *, cache_root: str | Path | None = None ) -> tuple[int, np.ndarray | None]: """A table's per-event constant and its read-only per-node counts, indexed by node ID. @@ -254,7 +255,7 @@ def read_token_lengths( if entry["file"] is None: return entry["constant"], None - path = Path(index, "col_stats", entry["file"]) + path = ColumnCache(index, cache_root).resolve(entry["file"]) if not path.stat().st_size: return entry["constant"], np.frombuffer(b"", dtype=" ColumnStats: - """The requested statistics at a cutoff, first computing into the index those it lacks.""" + """Read stored statistics, computing missing values into the selected cache root.""" import diskgraph - directory = Path(index, "col_stats", _cutoff_dir(ticks, unit)) - paths = { - table: directory / f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests - } + cache = ColumnCache(index, cache_root) + cutoff = _cutoff_dir(ticks, unit) + # Native publication locks, merges existing keys, and atomically replaces files. + # Sharing a cutoff directory lets overlapping requests reuse prior statistics. + relative = cutoff + paths = {table: f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests} + + files = set() def read() -> dict: - return { - table: json.loads(path.read_text())[table] - for table, path in paths.items() - if path.is_file() - } + found = {} + for table, name in paths.items(): + for candidate in cache.candidates(f"{relative}/{name}", base_name=f"{cutoff}/{name}"): + if candidate.is_file(): + files.add(candidate) + data = json.loads(candidate.read_text())[table] + for column, values in data.items(): + found.setdefault(table, {}).setdefault(column, {}).update(values) + return found found = read() missing = [ - (table, column, stype) - for table, column, stype in requests - if _STAT_KEYS[stype] not in found.get(table, {}).get(column, {}) + item + for item in requests + if _STAT_KEYS[item[2]] not in found.get(item[0], {}).get(item[1], {}) ] if missing: request = stats_request(missing, [ticks], unit) - diskgraph.cutoff_col_stats(str(index), json.dumps(request), num_threads=num_threads) + request["dirs"] = [relative] + options = {"output_dir": str(cache.root)} if cache.external else {} + diskgraph.cutoff_col_stats( + str(index), json.dumps(request), num_threads=num_threads, **options + ) found = read() return ColumnStats( {item: found[item[0]][item[1]] for item in requests}, {"value": ticks, "unit": unit}, - tuple(f"{directory.name}/{path.name}" for _, path in sorted(paths.items())), + tuple(cache.reference(path) for path in sorted(files)), ) diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py index 87eab16dd..b81d83a4c 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py @@ -5,13 +5,14 @@ """The engine interface: what an engine must provide, and how a caller picks one. A :class:`~kumo.pytorch.dataset.Dataset` says where the data is; -an :class:`Engine` says what reads it. Both are supplied by the caller, and -neither names the other. +an :class:`Engine` says what reads it. Source ingestion supplies both. +An existing-index ``DiskGraphEngine`` supplies the data binding itself, so +``prepare`` and ``check`` accept it without a ``Dataset``. An engine is a factory, not a holder of data: :meth:`Engine.open` binds a dataset and returns a :class:`SampleSource`. Construction-time configuration (artifact directories, index paths) belongs to the engine; data binding belongs -to the dataset. +to the dataset or the explicitly adopted index. Distributed execution is expressed by ``rank``/``world_size`` on ``Task.batches``. diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py index 1d7b7a10c..7456c398f 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py @@ -8,6 +8,7 @@ import datetime as dt import hashlib +import inspect import json import os import time @@ -21,6 +22,7 @@ import numpy as np import torch +from kumo.pytorch._storage import ColumnCache, normalize_work_dir from kumo.pytorch.anchors import AnchorTable from kumo.pytorch.column_stats import ( ColumnStats, @@ -73,6 +75,7 @@ from kumo.pytorch.engine import AnchorTask, GraphRoot, SampleSource, register_engine from kumo.pytorch.engines.parquet_anchors import ParquetAnchors, check_parquet_anchors from kumo.pytorch.engines.unsupported import DISKGRAPH_UNSUPPORTED, Unsupported +from kumo.pytorch.index_binding import open_index from kumo.pytorch.materialization import _DiskGraphIndex, ensure_diskgraph_index from kumo.pytorch.spec import DataLoaderSpec, ValidationIssue @@ -214,6 +217,16 @@ class DiskGraphEngine: requires an existing one. A mismatching manifest is an error, never permission to reuse. :param encoder_cutoffs: ``encoder_cutoff`` values whose column statistics an ingest computes. + :param work_dir: task/cache directory. Omit it or pass the index path (including + an alias resolving to it) for the original index-local layout and cache reuse. + Other paths must be outside the index; existing-index factories require an + external directory. + :param source_versions: optional table-to-identity mapping for source ingestion/reuse. + Each entry has nonempty ``source`` (connector plus source identity) and ``version`` + strings. These trusted connector versions replace file path/size/mtime identity; + missing versions are never treated as equal. Omit for legacy file validation. + The caller must change the version whenever source data changes: reuse does + not compare Parquet contents, file sizes or mtimes in this opt-in mode. """ name = "diskgraph" @@ -228,12 +241,21 @@ def __init__( num_shards: int = 1, ingest: Literal["if_missing", "never"] = "if_missing", encoder_cutoffs: Sequence[str | int | dt.datetime | dt.date | None] = (), + work_dir: str | Path | None = None, + source_versions: Mapping[str, Mapping[str, str]] | None = None, ): if ingest not in {"if_missing", "never"}: raise ValueError("ingest must be 'if_missing' or 'never'") if isinstance(encoder_cutoffs, str): raise TypeError("encoder_cutoffs is a list of cutoffs; wrap a single one in a list") self.index = Path(index) + self.work_dir = normalize_work_dir(self.index, work_dir) + self.source_versions = ( + None + if source_versions is None + else {name: dict(entry) for name, entry in source_versions.items()} + ) + self._binding = None self.num_threads = num_threads self.num_shards = num_shards self.ingest = ingest @@ -241,6 +263,68 @@ def __init__( dict.fromkeys(cutoff_nanos(value) for value in encoder_cutoffs) ) + @classmethod + def from_index( + cls, index: str | Path, *, work_dir: str | Path, num_threads: int | None = None + ) -> DiskGraphEngine: + """Open a complete native index without its original sources or any base writes. + + Read schema/relations from existing metadata. Older indexes without the + resolved native config are refused. Native edge tables and non-scalar + stored columns are not supported. Bare indexes without producer identity + are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan. + ``work_dir`` holds task extensions and derived caches outside the base. + """ + work_dir = normalize_work_dir(index, work_dir, external_only=True) + result = cls(index=index, ingest="never", work_dir=work_dir, num_threads=num_threads) + result._binding = open_index(index) + return result + + @classmethod + def from_snapshot( + cls, snapshot: Any, *, work_dir: str | Path, num_threads: int | None = None + ) -> DiskGraphEngine: + """Bind a kumo.engine GraphSnapshot using its existing index and identity. + + Uses public ``index_path`` and ``id`` fields; importing kumo.engine is + unnecessary. Load exported snapshots through the existing engine loader. + No source export, base ingestion or additional metadata file is needed. + """ + work_dir = normalize_work_dir(snapshot.index_path, work_dir, external_only=True) + result = cls( + index=snapshot.index_path, ingest="never", work_dir=work_dir, num_threads=num_threads + ) + result._binding = open_index(snapshot.index_path, snapshot_id=snapshot.id) + return result + + @property + def schema(self) -> DataSchema: + """Resolved stored schema for a from_index/from_snapshot engine.""" + if self._binding is None: + raise ValueError("schema is available on from_index/from_snapshot engines") + return self._binding.schema + + def _resolve_index_schema(self, schema: DataSchema) -> DataSchema: + if self._binding is None: + raise ValueError("dataset is required unless the engine binds an existing index") + return self._binding.resolve(schema)[0] + + def _check_binding(self, dataset: Dataset | None, schema: DataSchema): + if self._binding is not None and dataset is not None: + return ( + ValidationIssue( + "dataset", + "an existing-index engine does not accept Dataset", + "omit dataset; the index supplies the table binding", + ), + ) + if dataset is None: + try: + self._resolve_index_schema(schema) + except ValueError as error: + return (ValidationIssue("schema", str(error)),) + return () + def check_contract(self, contract: DataLoaderSpec) -> tuple[ValidationIssue, ...]: """Refuse the entries of `unsupported` the contract uses, before any data is read.""" issues = _sample_refusals(contract.sample) @@ -394,11 +478,38 @@ def check_anchors(table: AnchorTable) -> tuple[ValidationIssue, ...]: def open( self, - dataset: Dataset, + dataset: Dataset | None, schema: DataSchema, *, column_stats: Sequence[StatRequest] = (), ) -> SampleSource: + if self.work_dir is not None: + import diskgraph + + try: + inspect.signature(diskgraph.cutoff_col_stats).bind( + str(self.index), "{}", output_dir=str(self.work_dir) + ) + except (TypeError, ValueError) as error: + raise RuntimeError( + "External work_dir requires a DiskGraph build with " + "cutoff_col_stats(output_dir=...) support; rebuild/install DiskGraph " + "from the matching kumo-engine revision" + ) from error + if self._binding is not None: + if dataset is not None: + raise ValueError("an existing-index engine does not accept Dataset") + self._binding.validate_files() + schema, artifact = self._binding.resolve(schema) + return DiskGraphSource( + artifact, + schema, + num_threads=self.num_threads, + work_dir=self.work_dir, + identity=self._binding.identity, + ) + if dataset is None: + raise ValueError("dataset is required for source ingestion") ingests = bool(column_stats and self.encoder_cutoffs) and not self.index.exists() artifact = ensure_diskgraph_index( schema, @@ -408,11 +519,13 @@ def open( num_threads=self.num_threads, if_missing=self.ingest == "if_missing", column_stats=(column_stats, self.encoder_cutoffs) if ingests else None, + source_versions=self.source_versions, ) return DiskGraphSource( artifact, schema, num_threads=self.num_threads, + work_dir=self.work_dir, ) def materialize_anchors(self, table: AnchorTable) -> AnchorTask: @@ -430,6 +543,8 @@ def __init__( schema: DataSchema, *, num_threads: int | None, + work_dir: Path | None = None, + identity: str | None = None, ) -> None: try: import diskgraph @@ -445,6 +560,16 @@ def __init__( self.schema = schema self._index_path = artifact.path + self.artifact_identity = identity + self._work_dir = Path(artifact.path) if work_dir is None else work_dir + cache_id = ( + identity + or hashlib.sha256( + json.dumps(dict(artifact.manifest), sort_keys=True).encode() + ).hexdigest() + ) + self._column_cache = ColumnCache.for_work_dir(artifact.path, work_dir, cache_id) + self._cache_root = self._column_cache.root metadata = json.loads(Path(artifact.path, "index_metadata.json").read_text()) self._clock = metadata.get("timestamp_unit", "seconds") self._num_threads = num_threads @@ -1123,7 +1248,14 @@ def column_stats(self, requests: Sequence[StatRequest], cutoff: int | None) -> C if cutoff is None else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor") ) - return read_column_stats(self._index_path, requests, ticks, self._clock, self._num_threads) + return read_column_stats( + self._index_path, + requests, + ticks, + self._clock, + self._num_threads, + cache_root=self._cache_root, + ) def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[str, str]: ticks = ( @@ -1132,7 +1264,7 @@ def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[ else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor") ) name = encoder.tokenizer_file(ticks, self._clock) - path = Path(self._index_path, "col_stats", name) + path = self._column_cache.resolve(name) if path.is_file(): try: document = json.loads(path.read_text()) @@ -1181,17 +1313,19 @@ def token_lengths( else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor") ) name = token_lengths_file(table, columns, required, ticks, self._clock) - path = Path(self._index_path, "col_stats", name) + path = self._column_cache.resolve(name) if not path.is_file(): path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}") self._diskgraph.token_lengths( self._index_path, table, list(columns), - str(path), + str(temporary), required=list(required), num_threads=self._num_threads, ) + temporary.replace(path) return name def fit_graph_fields( diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index 32bb6ee96..cbb892d94 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -74,8 +74,8 @@ def __init__(self, source: Any, sample: Any, table: AnchorTable, *, directory=No self._source, self._sample, self._table = source, sample, table self._feature_types = sample_value_types(source.schema, sample) self._pa, self._pc = pa, pc - parent = Path(source._index_path) - (parent / "tasks").mkdir(exist_ok=True) + parent = Path(source._work_dir) + (parent / "tasks").mkdir(parents=True, exist_ok=True) (parent / "train_tables").mkdir(exist_ok=True) self._temporary = directory or tempfile.TemporaryDirectory( prefix="task-", dir=parent / "tasks" @@ -470,7 +470,9 @@ def bind_token_lengths(self, assembler, contract, statistics): raise ValueError( "event token costs do not match the designated input's fitted recipe" ) - fixed, values = read_token_lengths(self._source._index_path, entry) + fixed, values = read_token_lengths( + self._source._index_path, entry, cache_root=self._source._cache_root + ) if values is not None: self._token_maps.append(values) self._event_costs[table] = dict( diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py new file mode 100644 index 000000000..a830e0073 --- /dev/null +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py @@ -0,0 +1,279 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Read existing DiskGraph artifacts without their original table sources.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, replace +from pathlib import Path + +from kumo.pytorch.contract.dtypes import DType +from kumo.pytorch.contract.schema import NON_TEMPORAL, DataSchema, RelationSpec, TableSpec +from kumo.pytorch.materialization import _DiskGraphIndex, _fingerprint, _foreign_key + +_CLOCKS = {"seconds": "s", "milliseconds": "ms", "microseconds": "us", "nanoseconds": "ns"} + + +def _read(root: Path, name: str) -> dict: + try: + value = json.loads((root / name).read_text()) + except (OSError, ValueError) as error: + raise ValueError(f"cannot open DiskGraph metadata {root / name}: {error}") from error + if not isinstance(value, dict): + raise ValueError(f"DiskGraph metadata {root / name} must be an object") + return value + + +def _dtype(column: dict, clock: str, table: str) -> DType: + name = column["ingested_dtype"] + name = {"date": "date32", "utf8": "string"}.get(name, name.replace("u_int", "uint")) + try: + if name == "timestamp": + return DType(name, unit=_CLOCKS[clock], timezone="UTC") + if name.startswith("decimal"): + return DType(name, precision=column["precision"], scale=column["scale"]) + if column.get("semantic_type") == "multicategorical": + raise ValueError("multicategorical storage requires list support") + return DType(name) + except (KeyError, ValueError) as error: + raise ValueError( + f"unsupported stored type for {table}.{column['name']}: {column}" + ) from error + + +@dataclass(frozen=True) +class IndexBinding: + artifact: _DiskGraphIndex + schema: DataSchema + identity: str + metadata_fingerprint: str + physical_relations: tuple[tuple[RelationSpec, dict], ...] + + def resolve(self, schema: DataSchema) -> tuple[DataSchema, _DiskGraphIndex]: + """Validate a projection and map consumer relation aliases to physical edges.""" + stored = {table.name: table for table in self.schema.tables} + tables = [] + for table in schema.tables: + actual = stored.get(table.name) + if actual is None: + raise ValueError(f"index has no node table {table.name!r}") + if (table.primary_key, table.time_column) != (actual.primary_key, actual.time_column): + raise ValueError(f"index keys/clock differ for table {table.name!r}") + columns = actual.columns if table.columns is None else table.columns + resolved = {} + for name, dtype in columns.items(): + if name not in actual.columns: + raise ValueError(f"index has no column {table.name}.{name}") + expected = actual.columns[name] + if dtype is not None and dtype != expected: + raise ValueError( + f"index stored type differs for {table.name}.{name}: {expected}" + ) + resolved[name] = expected + tables.append(replace(table, columns=resolved)) + mapping = {} + definitions = {table.name: table for table in tables} + for relation in schema.relations: + try: + child, fk, parent, forward = _foreign_key(relation, definitions) + except KeyError as error: + raise ValueError( + f"relation {relation.name!r} refers to an undeclared table" + ) from error + matches = [ + native + for physical, native in self.physical_relations + if (physical.left_table, physical.left_key, physical.right_table) + == (child, fk, parent) + ] + if len(matches) != 1: + raise ValueError(f"index has no unambiguous stored relation for {relation.name!r}") + native = matches[0] + mapping[relation.name] = { + "left_table": relation.left_table, + "right_table": relation.right_table, + "forward": native["forward" if forward else "reverse"], + "reverse": native["reverse" if forward else "forward"], + } + return DataSchema(tuple(tables), schema.relations), _DiskGraphIndex( + self.artifact.path, {"relation_mapping": mapping} + ) + + def validate_files(self) -> None: + root = Path(self.artifact.path) + if _metadata_fingerprint(root) != self.metadata_fingerprint: + raise ValueError("DiskGraph index metadata changed after binding; open it again") + _check_complete(root) + + +def _check_complete(root: Path) -> None: + import diskgraph + + if (root / "_MANIFEST.json").is_file(): + diskgraph.verify_snapshot_integrity(str(root)) + elif (root / "index_manifest.json").is_file(): + # PyTorch publishes this manifest by atomic rename after native ingest; + # that producer does not request engine _MANIFEST/_SUCCESS markers. + _DiskGraphIndex.open(root) + else: + raise ValueError(f"incomplete DiskGraph index: {root} has no completion manifest") + + +def _metadata_fingerprint(root: Path) -> str: + return _fingerprint( + { + name: hashlib.sha256((root / name).read_bytes()).hexdigest() + for name in ( + "diskgraph_config.json", + "index_metadata.json", + "_MANIFEST.json", + "index_manifest.json", + "_provenance.json", + ) + if (root / name).is_file() + } + ) + + +def open_index(path: str | Path, *, snapshot_id: str | None = None) -> IndexBinding: + """Build an in-memory binding; write nothing and never follow stored source paths.""" + root = Path(path).resolve() + if not (root / "diskgraph_config.json").is_file(): + raise ValueError( + f"{root} has no diskgraph_config.json; automatic opening of this legacy " + "index is unsupported (no schema is inferred from unavailable sources)" + ) + _check_complete(root) + config, metadata = _read(root, "diskgraph_config.json"), _read(root, "index_metadata.json") + clock = metadata.get("timestamp_unit", "seconds") + if clock not in _CLOCKS or config.get("timestamp_unit", "seconds") != clock: + raise ValueError(f"unsupported or inconsistent DiskGraph clock: {clock!r}") + nodes = {node["name"]: node for node in metadata["node_types"]} + tables = [] + for table in config["tables"]: + name = table["name"] + if table.get("edge_table") is not None: + raise ValueError(f"native edge table {name!r} is not supported by the PyTorch binding") + node = nodes.get(name) + if node is None: + raise ValueError(f"config table {name!r} is absent from native node metadata") + key, time = table.get("primary_key"), table.get("create_time") + if key != (node.get("primary_key_map") or {}).get("column"): + raise ValueError(f"inconsistent native primary key for {name!r}") + if time != node.get("create_time_col") or table.get("end_time"): + raise ValueError(f"inconsistent or unsupported native clock/end_time for {name!r}") + tables.append( + TableSpec( + name, + key, + {col["name"]: _dtype(col, clock, name) for col in table["columns"]}, + time_column=time or NON_TEMPORAL, + ) + ) + if set(nodes) != {table.name for table in tables}: + raise ValueError("native node types differ from the resolved config") + by_name = {table.name: table for table in tables} + edges = {edge["name"]: edge for edge in metadata["edge_relations"]} + physical = [] + for table in config["tables"]: + for fk in table.get("fkeys", []): + child, column, parent = table["name"], fk["src_column"], fk["dst_table"] + if parent not in by_name or column not in by_name[child].columns: + raise ValueError(f"invalid native foreign key {child}.{column} -> {parent}") + pk = by_name[parent].primary_key + # Native FK naming is defined by ingest; verify the name AND endpoints/pair. + name = f"{child}_{column}_to_{parent}_{pk}" + edge = edges.get(name) + reverse = edges.get((edge or {}).get("reverse_of")) + if ( + edge is None + or reverse is None + or (edge["src_type"], edge["dst_type"]) != (child, parent) + or (reverse["src_type"], reverse["dst_type"]) != (parent, child) + or reverse.get("reverse_of") != name + ): + raise ValueError(f"missing or inconsistent native FK edge pair for {name!r}") + for direction in (edge, reverse): + target = by_name[direction["dst_type"]] + expected = ( + None + if target.time_column == NON_TEMPORAL + else (target.name, target.time_column) + ) + timestamp = (direction.get("dest_sort_key") or {}).get("timestamp") + actual = None if timestamp is None else (timestamp["table"], timestamp["column"]) + if actual != expected: + raise ValueError(f"unsupported stored edge clock for {direction['name']!r}") + physical.append( + ( + RelationSpec(name, child, column, parent, pk), + { + "left_table": child, + "right_table": parent, + "forward": name, + "reverse": reverse["name"], + }, + ) + ) + legacy = _DiskGraphIndex.open(root).manifest if (root / "index_manifest.json").is_file() else {} + relations = [] + for physical_relation, mapping in physical: + aliases = [ + (alias, entry) + for alias, entry in legacy.get("relation_mapping", {}).items() + if entry["forward"] in (mapping["forward"], mapping["reverse"]) + ] + if not aliases: + relations.append(physical_relation) + for alias, entry in aliases: + if entry["forward"] == mapping["forward"]: + relations.append(replace(physical_relation, name=alias)) + else: + relations.append( + RelationSpec( + alias, + physical_relation.right_table, + physical_relation.right_key, + physical_relation.left_table, + physical_relation.left_key, + ) + ) + schema = DataSchema(tuple(tables), tuple(relations)) + provenance = _read(root, "_provenance.json") if (root / "_provenance.json").is_file() else {} + recorded_id = provenance.get("id") + if snapshot_id and recorded_id and snapshot_id != recorded_id: + raise ValueError("snapshot identity differs from persisted provenance") + identity = snapshot_id or recorded_id + if not identity and legacy: + identity = _fingerprint( + { + key: legacy[key] + for key in ("dataset_fingerprint", "schema_fingerprint", "config_fingerprint") + } + ) + if not identity: + # Bare native indexes have no guaranteed producer identity. Hash the existing + # artifact once, rather than equating same-sized files or inventing metadata. + digest = hashlib.sha256() + for entry in sorted(_read(root, "_MANIFEST.json")["files"], key=lambda e: e["rel_path"]): + file = root / entry["rel_path"] + if not file.resolve().is_relative_to(root): + raise ValueError("native manifest contains a path outside the index") + digest.update(entry["rel_path"].encode()) + with file.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + identity = digest.hexdigest() + binding = IndexBinding( + _DiskGraphIndex(str(root), {}), + schema, + str(identity), + _metadata_fingerprint(root), + tuple(physical), + ) + _, artifact = binding.resolve(schema) + return replace(binding, artifact=artifact) diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py index fc698a929..aa79cf0ef 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py @@ -56,9 +56,9 @@ def open(cls, path: str | Path) -> _DiskGraphIndex: raise ValueError(f"{root} is missing DiskGraph index_metadata.json") return cls(str(root), manifest) - def validate_binding(self, schema: DataSchema, dataset: Dataset) -> None: + def validate_binding(self, schema: DataSchema, dataset: Dataset, source_versions=None) -> None: expected_schema = _fingerprint(_schema_payload(schema)) - expected_dataset = _fingerprint(_dataset_payload(dataset)) + expected_dataset = _fingerprint(_dataset_payload(dataset, source_versions)) if self.manifest.get("schema_fingerprint") != expected_schema: raise ValueError("DiskGraph index was materialized from a different schema") if self.manifest.get("dataset_fingerprint") != expected_dataset: @@ -75,6 +75,7 @@ def ensure_diskgraph_index( dataset_name: str = "default", if_missing: bool = True, column_stats: tuple[Sequence[StatRequest], Sequence[int | None]] | None = None, + source_versions: Mapping[str, Mapping[str, str]] | None = None, ) -> _DiskGraphIndex: """Reuse or build the base index while opening a DiskGraph engine. @@ -92,10 +93,16 @@ def ensure_diskgraph_index( if not dataset_name: raise ValueError("dataset_name must be non-empty") + for name, source in dataset.tables.items(): + if isinstance(source, Parquet) and source.columns: + raise DISKGRAPH_UNSUPPORTED["DG-COLUMN-RENAME"].error( + "DiskGraph materialization requires matching physical column names; " + f"table {name!r} declares {dict(source.columns)!r}" + ) output = Path(destination).resolve() if output.exists(): artifact = _DiskGraphIndex.open(output) - artifact.validate_binding(schema, dataset) + artifact.validate_binding(schema, dataset, source_versions) _warn_dropped_rows(output) return artifact if not if_missing: @@ -103,7 +110,10 @@ def ensure_diskgraph_index( output.parent.mkdir(parents=True, exist_ok=True) diskgraph = _import_diskgraph() - source_snapshot = _dataset_payload(dataset) + physical_snapshot = _dataset_payload(dataset) + source_snapshot = ( + physical_snapshot if source_versions is None else _dataset_payload(dataset, source_versions) + ) config, relation_mapping, source_types = _compile_diskgraph_config( schema, dataset, @@ -148,7 +158,7 @@ def ensure_diskgraph_index( **thread_options, ) elapsed = time.perf_counter() - started - if _dataset_payload(dataset) != source_snapshot: + if _dataset_payload(dataset) != physical_snapshot: raise ValueError("Parquet files changed while DiskGraph was ingesting them") manifest = { @@ -426,7 +436,20 @@ def _schema_payload(schema: DataSchema) -> dict[str, Any]: } -def _dataset_payload(dataset: Dataset) -> dict[str, Any]: +def _dataset_payload(dataset: Dataset, source_versions=None) -> dict[str, Any]: + if source_versions is not None: + if set(source_versions) != set(dataset.tables): + raise ValueError("source_versions must identify every dataset table exactly once") + result = {} + for name, entry in source_versions.items(): + if set(entry) != {"source", "version"} or any( + not isinstance(value, str) or not value for value in entry.values() + ): + raise ValueError( + "source_versions entries require nonempty source and version strings" + ) + result[name] = dict(entry) + return {"source_versions": result} payload = {} for name, source in sorted(dataset.tables.items()): if not isinstance(source, Parquet): diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py index 0d40d51d0..448de5184 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py @@ -69,8 +69,8 @@ def check( contract: DataLoaderSpec, - dataset: Dataset, - engine: Engine, + dataset: Dataset | None = None, + engine: Engine | None = None, *, anchors: AnchorTable | None = None, inference: bool = False, @@ -91,12 +91,14 @@ def check( def _check( contract: DataLoaderSpec, - dataset: Dataset, + dataset: Dataset | None, engine: Engine, *, anchors: AnchorTable | None = None, inference: bool = False, ) -> tuple[DataLoaderSpec, tuple[ValidationIssue, ...]]: + if engine is None: + raise TypeError("engine is required") entity_issues = _entity_key_issues(contract) if entity_issues: return contract, entity_issues @@ -105,6 +107,8 @@ def _check( try: # Keep reporting declaration issues for a bad binding when types are # already known, but never read metadata through a missing binding. + if dataset is None and not issues: + contract = replace(contract, schema=engine._resolve_index_schema(contract.schema)) contract = contract.resolve(dataset=None if issues else dataset) except (ValueError, KeyError, OSError) as error: if not issues: @@ -163,7 +167,7 @@ class Plan: def __init__( self, contract: DataLoaderSpec, - dataset: Dataset, + dataset: Dataset | None, engine: Engine, source: SampleSource, *, @@ -174,7 +178,9 @@ def __init__( self.engine = engine self.inference = inference self._source: SampleSource | None = source - self._dataset_fingerprint = _fingerprint(dataset) + self._dataset_fingerprint = getattr(source, "artifact_identity", None) or _fingerprint( + dataset + ) def materialize( self, @@ -272,8 +278,8 @@ def materialize( from kumo.pytorch.aggregate_materialization import materialize_values - parent = Path(source._index_path) / "tasks" - parent.mkdir(exist_ok=True) + parent = Path(source._work_dir) / "tasks" + parent.mkdir(parents=True, exist_ok=True) directory = tempfile.TemporaryDirectory(prefix="task-", dir=parent) try: values = materialize_values( @@ -574,12 +580,16 @@ def reused_state( def prepare( contract: DataLoaderSpec, - dataset: Dataset, - engine: Engine, + dataset: Dataset | None = None, + engine: Engine | None = None, *, inference: bool = False, ) -> Plan: - """Validate, then bind the dataset through the engine. + """Validate, then bind sources or an existing index through the engine. + + Omit ``dataset`` only for a DiskGraphEngine created with ``from_index`` or + ``from_snapshot``. Its stored schema supplies omitted types; it never reads + the original sources or rebuilds the base. ``engine`` is always required. Every check that does not need data happens first, so an unexecutable combination is refused before a row is read. Anchors, and everything fitted @@ -660,10 +670,17 @@ def preview( def _binding_issues( contract: DataLoaderSpec, - dataset: Dataset, + dataset: Dataset | None, engine: Engine, ) -> list[ValidationIssue]: issues: list[ValidationIssue] = [] + _check_binding = getattr(engine, "_check_binding", None) + if _check_binding is not None: + issues.extend(_check_binding(dataset, contract.schema)) + if dataset is None: + if _check_binding is None: + issues.append(ValidationIssue("dataset", "this engine requires a Dataset")) + return issues declared = [table.name for table in contract.schema.tables] missing = dataset.missing_for(declared) if missing: diff --git a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py index 50a645713..7eb3aa953 100644 --- a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py +++ b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py @@ -86,7 +86,14 @@ class Shape: class _Source: """In place of DiskGraphSource: what the engine hands the sampler.""" - def __init__(self, artifact: _DiskGraphIndex, schema: DataSchema, *, num_threads: int | None): + def __init__( + self, + artifact: _DiskGraphIndex, + schema: DataSchema, + *, + num_threads: int | None, + work_dir: Path | None = None, + ): self.artifact, self.schema, self.num_threads = artifact, schema, num_threads @@ -381,7 +388,11 @@ def _resize(path: Path) -> None: @pytest.mark.parametrize("change", ("mtime", "size", "extra_file")) def test_a_changed_dataset_is_refused_not_reused(change: str, tmp_path: Path) -> None: - """M06: the dataset fingerprint covers each file's path, size and mtime.""" + """M06: source ingestion without connector versions validates path/size/mtime. + + Explicit existing-index adoption and version-aware restaging are covered + by test_existing_index; those modes do not require the original files. + """ with _built(tmp_path, parts=2) as (dataset, index): before = _snapshot(index) diff --git a/packages/experimental/kumo-pytorch/tests/test_existing_index.py b/packages/experimental/kumo-pytorch/tests/test_existing_index.py new file mode 100644 index 000000000..56c947f12 --- /dev/null +++ b/packages/experimental/kumo-pytorch/tests/test_existing_index.py @@ -0,0 +1,697 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Source-free, read-only reuse through the public preparation lifecycle.""" + +import hashlib +import json +import shutil +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from kumo.pytorch import api +from kumo.pytorch.column_stats import TableColumnEncoder, read_token_lengths + +diskgraph = pytest.importorskip("diskgraph") + + +def inventory(root): + return { + str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in root.rglob("*") + if p.is_file() + } + + +@contextmanager +def read_only(root): + paths = [root, *root.rglob("*")] + modes = {p: p.stat().st_mode for p in paths} + for p in paths: + p.chmod(0o555 if p.is_dir() else 0o444) + try: + yield + finally: + for p, mode in modes.items(): + p.chmod(mode) + + +def contract(schema): + return api.DataLoaderSpec( + schema, + api.SampleSpec( + entity=api.EntitySpec("users"), + neighborhoods={ + "history": api.NeighborhoodSpec( + (api.GraphLayer((api.GraphSample("orders", fanout=2, strategy="recent"),)),) + ), + }, + ), + api.BatchSpec( + inputs={ + "graph": api.Input( + api.NeighborhoodSource("history"), + api.Graph( + node_fields={ + table.name: api.Fields( + tuple(api.Encode(name, api.Raw()) for name in table.columns) + ) + for table in schema.tables + } + ), + None, + ) + }, + batching=api.Batching(max_samples=2), + ), + ) + + +@pytest.fixture +def indexed(tmp_path): + source = tmp_path / "sources" + source.mkdir() + data = { + "users": pa.table({"id": [-2, 7], "name": ["Alice", "Bob"]}), + "events": pa.table( + { + "id": ["a", "b", "c", "d"], + "user": [-2, -2, -2, 7], + "reviewer": [7, 7, 7, -2], + "parent": pa.array([None, "a", "b", None], pa.string()), + "at": pa.array([10, 20, 30, 10], pa.timestamp("s")), + "value": [2.0, 4.0, 100.0, None], + "note": ["first item", "second item", "future item", None], + } + ), + } + for name, table in data.items(): + pq.write_table(table, source / f"{name}.parquet") + schema = api.DataSchema( + tuple( + api.TableSpec( + name, + "id", + {f.name: api.DType.from_arrow(f.type) for f in table.schema}, + time_column="at" if name == "events" else api.NON_TEMPORAL, + ) + for name, table in data.items() + ), + ( + api.RelationSpec("orders", "users", "id", "events", "user"), + api.RelationSpec("reviews", "events", "reviewer", "users", "id"), + api.RelationSpec("parent", "events", "parent", "events", "id"), + ), + ) + dataset = api.Dataset({name: api.Parquet(str(source / f"{name}.parquet")) for name in data}) + index = tmp_path / "index" + spec = contract(schema) + with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1)): + pass + anchor_path = tmp_path / "anchors.parquet" + pq.write_table( + pa.table( + { + "user": [-2, 7], + "at": pa.array([20, 20], pa.timestamp("s")), + "sample": ["left", "right"], + } + ), + anchor_path, + ) + anchors = api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at", "sample") + return index, source, dataset, spec, anchors + + +def sample(engine, spec, anchors): + with api.prepare(contract=spec, engine=engine) as plan: + with plan.materialize(anchors) as task: + graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"] + events = graph.nodes["events"] + assert set(events.ids.to_pylist()) == {"a", "b", "d"} + assert events.features["at"].type == pa.timestamp("s", tz="UTC") + assert set(events.features["value"].to_pylist()) == {2.0, 4.0, None} + assert graph.edges["orders"].source_type == "users" + assert graph.edges["orders"].destination_type == "events" + for left, right in zip( + *[graph.edges["orders"].index[n].to_pylist() for n in ("source", "destination")], + strict=True, + ): + assert graph.nodes["users"].owners[left] == events.owners[right] + return graph + + +def test_relocated_index_without_sources_or_base_writes(indexed, tmp_path, monkeypatch): + index, source, dataset, spec, anchors = indexed + relocated = tmp_path / "relocated" + shutil.move(str(index), relocated) + shutil.rmtree(source) + before = inventory(relocated) + original_ingest = diskgraph.ingest + extensions = [] + + def ingest(*args, **kwargs): + assert kwargs.get("parent_index_path") == str(relocated) + assert not str(args[1]).startswith(str(relocated)) + extensions.append(args[1]) + return original_ingest(*args, **kwargs) + + monkeypatch.setattr(diskgraph, "ingest", ingest) + with read_only(relocated): + engine = api.DiskGraphEngine.from_index( + relocated, work_dir=tmp_path / "work", num_threads=1 + ) + assert engine.schema == spec.schema + sample(engine, spec, anchors) + assert extensions + spec_path = tmp_path / "contract.json" + spec_path.write_text(spec.to_json()) + subprocess.run( + [ + sys.executable, + "-c", + """ +import sys +from pathlib import Path +from kumo.pytorch import api +index, work, spec, anchors = sys.argv[1:] +engine = api.DiskGraphEngine.from_index(index, work_dir=work, num_threads=1) +contract = api.DataLoaderSpec.from_json(Path(spec).read_text()) +with api.prepare(contract=contract, engine=engine) as plan: + with plan.materialize(api.AnchorTable(api.Parquet(anchors), 'user', 'at', 'sample')) as task: + graph = next(iter(task.batches(output_format='columnar'))).inputs['graph'] + assert set(graph.nodes['events'].ids.to_pylist()) == {'a', 'b', 'd'} +""", + str(relocated), + str(tmp_path / "fresh-process"), + str(spec_path), + anchors.source.path, + ], + check=True, + capture_output=True, + text=True, + ) + assert inventory(relocated) == before + assert all(not Path(path).exists() for path in extensions) + assert not (relocated / "tasks").exists() + + +def test_snapshot_binding_needs_no_pytorch_manifest(indexed, tmp_path): + index, source, _, spec, anchors = indexed + config = (index / "diskgraph_config.json").read_text() + index = tmp_path / "engine-index" + diskgraph.ingest(config, str(index), col_stats=False, success_marker_content="{}") + # The real engine integration test covers connector -> GraphSnapshot creation. + snapshot = SimpleNamespace(index_path=index, id="graph@version", graph_config={}) + engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=tmp_path / "work", num_threads=1) + shutil.rmtree(source) + before = inventory(index) + sample(engine, spec, anchors) # Custom logical relation aliases resolve by endpoint AND key. + assert inventory(index) == before + + +def test_external_fitting_statistics_tokenizer_and_lengths(indexed, tmp_path): + index, _, _, spec, _ = indexed + before = inventory(index) + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1) + with read_only(index), api.prepare(contract=spec, engine=engine) as plan: + source = plan._source + stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9) + assert stats.columns[("events", "value", "numerical")]["count"] == 3 + assert stats.columns[("events", "value", "numerical")]["mean"] == 3.0 + encoder = TableColumnEncoder( + api.Text(vocabulary_size=300, min_frequency=1), + api.DType("string"), + (("events", "note"),), + ) + tokenizer, name = source.column_text(encoder, 20 * 10**9) + assert source.column_text(encoder, 20 * 10**9) == (tokenizer, name) + file = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9) + _, values = read_token_lengths( + index, {"file": file, "constant": 0}, cache_root=source._cache_root + ) + assert len(values) == 4 + assert inventory(index) == before + + +def test_existing_base_statistics_are_reused_and_reported(indexed, tmp_path, monkeypatch): + index, _, _, spec, _ = indexed + from kumo.pytorch.column_stats import read_column_stats + + requests = [("events", "value", "numerical")] + stored = read_column_stats(index, requests, 20, "seconds") + before = inventory(index) + + def unexpected_compute(*args, **kwargs): + pytest.fail("the requested statistics already exist in the base") + + monkeypatch.setattr(diskgraph, "cutoff_col_stats", unexpected_compute) + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work") + with read_only(index), api.prepare(contract=spec, engine=engine) as plan: + reused = plan._source.column_stats(requests, 20 * 10**9) + assert reused.columns == stored.columns + assert reused.files == tuple(str(index / "col_stats" / f) for f in stored.files) + assert inventory(index) == before + + +def test_external_statistics_compute_only_missing_requests(indexed, tmp_path, monkeypatch): + index, _, _, spec, _ = indexed + from kumo.pytorch.column_stats import read_column_stats + + known = ("events", "value", "numerical") + missing = ("events", "value", "categorical") + stored = read_column_stats(index, [known], 20, "seconds") + before = inventory(index) + calls = [] + original = diskgraph.cutoff_col_stats + + def compute(index_path, request_json, **options): + request = json.loads(request_json) + assert request["columns"] == {"events": {"value": ["categorical"]}} + assert not Path(options["output_dir"]).is_relative_to(index) + calls.append(request) + return original(index_path, request_json, **options) + + monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute) + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1) + with read_only(index), api.prepare(contract=spec, engine=engine) as plan: + source = plan._source + with ThreadPoolExecutor(max_workers=2) as pool: + results = list( + pool.map(lambda _: source.column_stats([known, missing], 20 * 10**9), range(2)) + ) + for result in results: + assert result.columns[known]["mean"] == stored.columns[known]["mean"] + assert sorted(result.columns[missing]["category_counts"]) == [["2", 1], ["4", 1]] + count = len(calls) + assert count >= 1 + assert source.column_stats([known, missing], 20 * 10**9) == results[0] + assert len(calls) == count + assert inventory(index) == before + + +@pytest.mark.parametrize( + "unit,clock", [("ms", "milliseconds"), ("us", "microseconds"), ("ns", "nanoseconds")] +) +def test_native_clock_precision_preserves_cutoff(indexed, tmp_path, unit, clock): + index, source, _, spec, _ = indexed + path = source / "events.parquet" + table = pq.read_table(path) + table = table.set_column( + table.schema.get_field_index("at"), + "at", + pa.array([1001, 1002, 1003, 1001], pa.timestamp(unit)), + ) + pq.write_table(table, path) + config = json.loads((index / "diskgraph_config.json").read_text()) + config["timestamp_unit"] = clock + native = tmp_path / "native" + diskgraph.ingest(json.dumps(config), str(native), col_stats=False, success_marker_content="{}") + shutil.rmtree(source) + anchor_path = tmp_path / "precise-anchors.parquet" + pq.write_table( + pa.table({"user": [-2, 7], "at": pa.array([1002, 1002], pa.timestamp(unit))}), anchor_path + ) + engine = api.DiskGraphEngine.from_index(native, work_dir=tmp_path / "work", num_threads=1) + schema = replace(engine.schema, relations=spec.schema.relations) + with api.prepare(contract=contract(schema), engine=engine) as plan: + with plan.materialize(api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at")) as task: + graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"] + events = graph.nodes["events"] + assert set(events.ids.to_pylist()) == {"a", "b", "d"} + assert events.features["at"].type == pa.timestamp(unit, tz="UTC") + assert set(events.features["at"].cast(pa.int64()).to_pylist()) == {1001, 1002} + + +def test_concurrent_consumers_keep_task_cleanup_separate(indexed, tmp_path): + index, _, _, spec, anchors = indexed + before = inventory(index) + + def run(_): + return sample( + api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "shared", num_threads=1), + spec, + anchors, + ) + + with ThreadPoolExecutor(max_workers=2) as pool: + assert len(list(pool.map(run, range(2)))) == 2 + assert inventory(index) == before + + +def test_projection_alias_validation_and_bad_bindings(indexed, tmp_path): + index, _, dataset, spec, anchors = indexed + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work") + assert api.check(spec, engine=engine) == () + with pytest.raises(api.ValidationError, match="does not accept Dataset"): + api.prepare(spec, dataset, engine) + unresolved = replace( + spec, + schema=replace( + spec.schema, + tables=tuple( + replace(table, columns={name: None for name in table.columns}) + for table in spec.schema.tables + ), + ), + ) + with api.prepare(contract=unresolved, engine=engine) as plan: + assert plan.contract.schema == spec.schema + wrong = replace(spec.schema.relations[0], right_key="reviewer") + projected = replace(spec.schema, relations=(wrong,)) + with api.prepare(contract=contract(projected), engine=engine) as plan: + with plan.materialize(anchors) as task: + graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"] + edge = graph.edges["orders"] + pairs = { + (graph.nodes["users"].ids[left].as_py(), graph.nodes["events"].ids[right].as_py()) + for left, right in zip( + edge.index["source"].to_pylist(), + edge.index["destination"].to_pylist(), + strict=True, + ) + } + assert pairs == {(-2, "d"), (7, "a"), (7, "b")} + missing = replace(wrong, right_key="value") + assert api.check( + replace(spec, schema=replace(spec.schema, relations=(missing,))), engine=engine + ) + with pytest.raises(ValueError, match="outside"): + api.DiskGraphEngine.from_index(index, work_dir=index / "work") + + +def test_physically_present_self_relation(indexed, tmp_path): + index, source, _, spec, anchors = indexed + shutil.rmtree(source) + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1) + spec = replace( + spec, + sample=replace( + spec.sample, + neighborhoods={ + "history": api.NeighborhoodSpec( + ( + api.GraphLayer((api.GraphSample("orders", fanout=1, strategy="recent"),)), + api.GraphLayer((api.GraphSample("parent", fanout=1, strategy="recent"),)), + ) + ), + }, + ), + ) + with api.prepare(contract=spec, engine=engine) as plan: + with plan.materialize(anchors) as task: + graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"] + events, edge = graph.nodes["events"], graph.edges["parent"] + assert { + (events.ids[left].as_py(), events.ids[right].as_py()) + for left, right in zip( + edge.index["source"].to_pylist(), + edge.index["destination"].to_pylist(), + strict=True, + ) + } == {("b", "a")} + + +@pytest.mark.parametrize( + "change,match", + [ + ("legacy", "diskgraph_config"), + ("incomplete", "completion manifest"), + ("edge", "edge table"), + ("list", "unsupported stored type"), + ("pair", "edge pair"), + ], +) +def test_incompatible_artifacts_fail_before_sampling(indexed, tmp_path, change, match): + index, _, _, _, _ = indexed + if change == "legacy": + (index / "diskgraph_config.json").unlink() + elif change == "incomplete": + (index / "index_manifest.json").unlink() + else: + filename = "index_metadata.json" if change == "pair" else "diskgraph_config.json" + path = index / filename + value = json.loads(path.read_text()) + if change == "edge": + value["tables"][0]["edge_table"] = {"src_table": "users", "dst_table": "users"} + elif change == "list": + value["tables"][0]["columns"][0]["ingested_dtype"] = "string_array" + else: + value["edge_relations"][0]["reverse_of"] = "absent" + path.write_text(json.dumps(value)) + # Keep the native size inventory internally consistent for this semantic refusal. + manifest_path = index / "_MANIFEST.json" + if manifest_path.exists(): + manifest = json.loads(manifest_path.read_text()) + for entry in manifest["files"]: + if entry["rel_path"] == filename: + entry["size_bytes"] = path.stat().st_size + manifest_path.write_text(json.dumps(manifest)) + with pytest.raises((ValueError, RuntimeError), match=match): + api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work") + + +def test_connector_versions_allow_restage_but_reject_changes(indexed, tmp_path): + legacy, source, dataset, spec, _ = indexed + versions = { + name: {"source": f"connector/catalog/{name}", "version": "v1"} for name in dataset.tables + } + path = tmp_path / "versioned" + with api.prepare(spec, dataset, api.DiskGraphEngine(index=path, source_versions=versions)): + pass + # Version-aware reuse explicitly trusts the connector, even if its token is stale. + # A caller that needs file-change detection must keep the default legacy binding. + event_path = source / "events.parquet" + events = pq.read_table(event_path) + pq.write_table( + events.set_column( + events.schema.get_field_index("value"), "value", pa.array([200.0, 400.0, 10000.0, None]) + ), + event_path, + ) + with pytest.raises(ValueError, match="different dataset"): + api.prepare(spec, dataset, api.DiskGraphEngine(index=legacy)) + moved = tmp_path / "new_sources" + shutil.move(source, moved) + restaged = api.Dataset( + {name: api.Parquet(str(moved / f"{name}.parquet")) for name in dataset.tables} + ) + before = inventory(path) + with api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=versions)): + pass + assert inventory(path) == before + for field in ("source", "version"): + changed = {name: {**entry, field: "different"} for name, entry in versions.items()} + with pytest.raises(ValueError, match="different dataset"): + api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=changed)) + with pytest.raises(ValueError, match="every dataset table"): + api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions={})) + + +@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink"]) +@pytest.mark.parametrize("explicit_first", [False, True]) +def test_explicit_index_work_dir_reuses_legacy_state( + indexed, tmp_path, monkeypatch, alias, explicit_first +): + index, _, dataset, spec, anchors = indexed + if alias == "relative": + monkeypatch.chdir(tmp_path) + work = Path("index") + elif alias == "symlink": + work = tmp_path / "index-alias" + work.symlink_to(index, target_is_directory=True) + else: + work = index + encoder = TableColumnEncoder( + api.Text(vocabulary_size=300, min_frequency=1), + api.DType("string"), + (("events", "note"),), + ) + + def run(explicit): + options = {"work_dir": work} if explicit else {} + engine = api.DiskGraphEngine(index=index, num_threads=1, **options) + with api.prepare(spec, dataset, engine) as plan: + source = plan._source + stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9) + tokenizer, name = source.column_text(encoder, 20 * 10**9) + lengths = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9) + _, values = read_token_lengths(index, {"file": lengths, "constant": 0}) + assert all((index / "col_stats" / path).is_file() for path in stats.files) + assert (index / "col_stats" / name).is_file() + with plan.materialize(anchors) as task: + assert any((index / "train_tables").iterdir()) + assert any((index / "tasks").iterdir()) + graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"] + assert set(graph.nodes["events"].ids.to_pylist()) == {"a", "b", "d"} + assert not list((index / "train_tables").iterdir()) + assert not list((index / "tasks").iterdir()) + return stats, tokenizer, name, lengths, values.tolist() + + expected = run(explicit_first) + before = inventory(index) + + def unexpected_compute(*args, **kwargs): + pytest.fail("existing in-index cache must be reused") + + for function in ("cutoff_col_stats", "train_tokenizer", "token_lengths"): + monkeypatch.setattr(diskgraph, function, unexpected_compute) + assert run(not explicit_first) == expected + assert inventory(index) == before + assert not (index / "cache").exists() + + +@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink", "nested"]) +def test_adopted_indexes_require_external_work_dir(indexed, tmp_path, monkeypatch, alias): + index, _, _, _, _ = indexed + if alias == "relative": + monkeypatch.chdir(tmp_path) + work = Path("index") + elif alias == "symlink": + work = tmp_path / "index-alias" + work.symlink_to(index, target_is_directory=True) + else: + work = index if alias == "absolute" else index / "work" + with pytest.raises(ValueError, match="outside"): + api.DiskGraphEngine.from_index(index, work_dir=work) + with pytest.raises(ValueError, match="outside"): + api.DiskGraphEngine.from_snapshot( + SimpleNamespace(index_path=index, id="test"), work_dir=work + ) + if alias == "nested": + with pytest.raises(ValueError, match="outside"): + api.DiskGraphEngine(index=index, work_dir=work) + + +@pytest.mark.parametrize("root", ["tasks", "train_tables", "cache"]) +@pytest.mark.parametrize("symlink", [False, True]) +def test_external_writable_roots_cannot_overlap_base(indexed, tmp_path, root, symlink): + index, _, _, _, _ = indexed + work = tmp_path / "work" + work.mkdir() + if symlink: + (work / root).symlink_to(index, target_is_directory=True) + else: + moved = work / root + shutil.move(index, moved) + index = moved + before = inventory(index) + with pytest.raises(ValueError, match="outside"): + api.DiskGraphEngine.from_index(index, work_dir=work) + assert inventory(index) == before + assert set(p.name for p in work.iterdir()) == {root} + + +def test_external_statistics_reuse_overlapping_requests(indexed, tmp_path, monkeypatch): + index, _, _, spec, _ = indexed + known = ("events", "value", "numerical") + missing = ("events", "value", "categorical") + engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1) + with api.prepare(contract=spec, engine=engine) as plan: + source = plan._source + initial = source.column_stats([known], 20 * 10**9) + original = diskgraph.cutoff_col_stats + calls = [] + + def compute(index_path, request_json, **options): + request = json.loads(request_json) + assert request["columns"] == {"events": {"value": ["categorical"]}} + calls.append(request) + return original(index_path, request_json, **options) + + monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute) + result = source.column_stats([known, missing], 20 * 10**9) + assert result.columns[known]["mean"] == initial.columns[known]["mean"] + assert result.files == initial.files + assert len(calls) == 1 + assert source.column_stats([missing], 20 * 10**9).files == initial.files + assert len(calls) == 1 + + +def test_trusted_reuse_refuses_column_renames(indexed, tmp_path): + _, _, dataset, spec, _ = indexed + versions = {name: {"source": name, "version": "v1"} for name in dataset.tables} + index = tmp_path / "trusted" + with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, source_versions=versions)): + pass + renamed = api.Dataset( + { + name: api.Parquet(source.path, columns={"value": "renamed"}) + if name == "events" + else source + for name, source in dataset.tables.items() + } + ) + with pytest.raises(ValueError, match="DG-COLUMN-RENAME"): + api.prepare(spec, renamed, api.DiskGraphEngine(index=index, source_versions=versions)) + + +def test_legacy_ingest_checks_the_snapshot_recorded_in_provenance(indexed, tmp_path, monkeypatch): + from kumo.pytorch import materialization + + _, source, dataset, spec, _ = indexed + original = materialization._dataset_payload + calls = 0 + + def capture(*args, **kwargs): + nonlocal calls + payload = original(*args, **kwargs) + calls += 1 + if calls == 1: + path = source / "events.parquet" + table = pq.read_table(path) + pq.write_table(table.slice(0, 2), path) + return payload + + monkeypatch.setattr(materialization, "_dataset_payload", capture) + index = tmp_path / "new-index" + with pytest.raises(ValueError, match="changed while DiskGraph was ingesting"): + api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1)) + assert not index.exists() + + +def test_external_storage_refuses_old_native_api_before_task_writes(indexed, tmp_path, monkeypatch): + index, _, dataset, spec, _ = indexed + + def old_cutoff_col_stats(index_dir, request_json, num_threads=None): + pytest.fail("compatibility check must not execute statistics") + + monkeypatch.setattr(diskgraph, "cutoff_col_stats", old_cutoff_col_stats) + work = tmp_path / "work" + engine = api.DiskGraphEngine.from_index(index, work_dir=work) + with pytest.raises(RuntimeError, match="matching kumo-engine revision"): + api.prepare(contract=spec, engine=engine) + assert not work.exists() + with api.prepare(spec, dataset, api.DiskGraphEngine(index=index)): + pass + + +def test_index_local_statistics_read_each_file_once(indexed, tmp_path, monkeypatch): + from kumo.pytorch.column_stats import read_column_stats + + index, _, _, _, _ = indexed + request = [("events", "value", "numerical")] + expected = read_column_stats(index, request, 20, "seconds") + alias = tmp_path / "alias" + alias.symlink_to(index, target_is_directory=True) + original = Path.read_text + reads = [] + + def read(path, *args, **kwargs): + if path.name.endswith(".col_stats.json"): + reads.append(path) + return original(path, *args, **kwargs) + + monkeypatch.setattr(Path, "read_text", read) + assert read_column_stats(index, request, 20, "seconds", cache_root=alias) == expected + assert len(reads) == 1 diff --git a/packages/kumo-engine/test/test_pytorch_snapshot.py b/packages/kumo-engine/test/test_pytorch_snapshot.py new file mode 100644 index 000000000..6df130c83 --- /dev/null +++ b/packages/kumo-engine/test/test_pytorch_snapshot.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Optional cross-package qualification of the installed PyTorch loader.""" + +import hashlib +import shutil + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from kumoapi.graph import Edge +from kumoapi.typing import Stype + +import kumo.engine as kumo +from kumo.engine._scratch import get_scratch_dir + +api = pytest.importorskip('kumo.pytorch.api') + + +def _files(root): + return { + str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in root.rglob('*') if p.is_file() + } + + +def test_connector_snapshot_columnar_export_reload(tmp_path, monkeypatch): + data = tmp_path / 'sources' + data.mkdir() + pq.write_table(pa.table({ + 'id': [-2, 7], + 'name': ['Alice', 'Bob'] + }), data / 'users.parquet') + pq.write_table( + pa.table({ + 'id': [1, 2, 3, 4], + 'user': [-2, -2, -2, 7], + 'at': pa.array([10, 20, 30, 10], pa.timestamp('s')), + 'value': [2.0, 4.0, 100.0, None], + }), + data / 'events.parquet', + ) + connector = kumo.FileConnector(root_dir=str(data)) + graph = kumo.Graph( + tables={ + 'users': + kumo.Table.from_source_table(connector['users.parquet'], + primary_key='id'), + 'events': + kumo.Table.from_source_table(connector['events.parquet'], + primary_key='id', + time_column='at').with_column( + 'user', stype=Stype.ID), + }, + edges=[Edge(src_table='events', fkey='user', dst_table='users')], + ) + previous = get_scratch_dir() + try: + kumo.configure(scratch_dir=tmp_path / 'producer') + snapshot = graph.snapshot() + assert not (snapshot.index_path / 'index_manifest.json').exists() + anchor_path = tmp_path / 'anchors.parquet' + pq.write_table( + pa.table({ + 'user': [-2, 7], + 'at': pa.array([20, 20], pa.timestamp('s')) + }), + anchor_path, + ) + anchors = api.AnchorTable(api.Parquet(str(anchor_path)), 'user', 'at') + + def run(snapshot, work): + before = _files(snapshot.index_path) + engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=work, + num_threads=1) + relation = engine.schema.relations[0] + contract = api.DataLoaderSpec( + engine.schema, + api.SampleSpec( + entity=api.EntitySpec('users'), + neighborhoods={ + 'history': + api.NeighborhoodSpec((api.GraphLayer((api.GraphSample( + relation.name, + fanout=2, + direction='reverse', + strategy='recent', + ), )), )), + }, + ), + api.BatchSpec( + inputs={ + 'graph': + api.Input( + api.NeighborhoodSource('history'), + api.Graph( + node_fields={ + table.name: + api.Fields( + tuple( + api.Encode(name, api.Raw()) + for name in table.columns)) + for table in engine.schema.tables + }), + None, + ) + }, + batching=api.Batching(max_samples=2), + ), + ) + with ( + api.prepare(contract=contract, engine=engine) as plan, + plan.materialize(anchors) as task, + ): + batch = next(iter(task.batches(output_format='columnar'))) + out = batch.inputs['graph'] + events = out.nodes['events'] + assert set(events.ids.to_pylist()) == {1, 2, 4} + assert set( + events.features['value'].to_pylist()) == {2.0, 4.0, None} + assert relation.name in out.edges + edge = out.edges[relation.name] + pairs = {(out.nodes[edge.source_type].ids[left].as_py(), + out.nodes[edge.destination_type].ids[right].as_py()) + for left, right in zip( + edge.index['source'].to_pylist(), + edge.index['destination'].to_pylist(), strict=True) + } + expected_pairs = {(-2, 1), (-2, 2), (7, 4)} + if edge.source_type == 'events': + expected_pairs = {(event, user) + for user, event in expected_pairs} + assert pairs == expected_pairs + for edge in out.edges.values(): + for left, right in zip( + edge.index['source'].to_pylist(), + edge.index['destination'].to_pylist(), + strict=True, + ): + assert (out.nodes[edge.source_type].owners[left] == + out.nodes[edge.destination_type].owners[right]) + result = events.features.to_pylist() + assert _files(snapshot.index_path) == before + return result, engine._binding.identity + + expected, identity = run(snapshot, tmp_path / 'work1') + exports = kumo.FileConnector(root_dir=str(tmp_path / 'exports')) + location = snapshot.export(kumo.SnapshotOutputConfig(exports, 'saved')) + shutil.rmtree(data) + shutil.rmtree(tmp_path / 'producer') + kumo.configure(scratch_dir=tmp_path / 'consumer') + loaded = kumo.GraphSnapshot.load_from_connector(exports, path=location) + import diskgraph + + original = diskgraph.ingest + + def extension_only(*args, **kwargs): + assert kwargs.get('parent_index_path') == str(loaded.index_path) + return original(*args, **kwargs) + + monkeypatch.setattr(diskgraph, 'ingest', extension_only) + assert run(loaded, tmp_path / 'work2') == (expected, identity) + assert (api.DiskGraphEngine.from_index( + loaded.index_path, + work_dir=tmp_path / 'work3')._binding.identity == identity) + finally: + kumo.configure(scratch_dir=previous)