diff --git a/.github/workflows/kumo-pytorch-diskgraph.yml b/.github/workflows/kumo-pytorch-diskgraph.yml
index e2d4bbcc2..3e312b62a 100644
--- a/.github/workflows/kumo-pytorch-diskgraph.yml
+++ b/.github/workflows/kumo-pytorch-diskgraph.yml
@@ -13,7 +13,13 @@ name: kumo-pytorch DiskGraph
on: # yamllint disable-line rule:truthy
pull_request:
paths:
+ - 'pyproject.toml'
- 'packages/experimental/kumo-pytorch/**'
+ - 'packages/kumo-engine/**'
+ - 'packages/kumo-core/**'
+ - 'packages/kumo-api/**'
+ - 'packages/kumo-pql/**'
+ - 'packages/kumo-settings/**'
- 'packages/diskgraph/**'
- 'scripts/diskgraph'
- '.github/workflows/_diskgraph-wheel.yml'
@@ -22,7 +28,13 @@ on: # yamllint disable-line rule:truthy
push:
branches: [main]
paths:
+ - 'pyproject.toml'
- 'packages/experimental/kumo-pytorch/**'
+ - 'packages/kumo-engine/**'
+ - 'packages/kumo-core/**'
+ - 'packages/kumo-api/**'
+ - 'packages/kumo-pql/**'
+ - 'packages/kumo-settings/**'
- 'packages/diskgraph/**'
- 'scripts/diskgraph'
- '.github/workflows/_diskgraph-wheel.yml'
@@ -150,3 +162,26 @@ jobs:
run: |
../fresh/bin/python -m pytest -p no:cacheprovider -m notebook --require-imports -q \
--durations=0 tests/test_notebooks.py
+ - name: Build the engine and its workspace dependencies as wheels
+ working-directory: .
+ run: |
+ set -euo pipefail
+ for package in kumo-api kumo-pql kumo-settings kumo-core kumo-engine; do
+ uv build --wheel --package "$package" --out-dir "$RUNNER_TEMP/engine-wheels"
+ done
+ cp packages/kumo-engine/test/test_pytorch_snapshot.py "$RUNNER_TEMP/stage/tests/"
+ - name: Install the engine wheels alongside PyTorch and DiskGraph
+ working-directory: ${{ runner.temp }}
+ run: |
+ uv pip install --python fresh/bin/python --torch-backend cpu engine-wheels/*.whl
+ uv pip check --python fresh/bin/python
+ - name: Connector snapshot to columnar batches, from installed wheels
+ working-directory: ${{ runner.temp }}/stage
+ run: |
+ ../fresh/bin/python -I -c '
+ import pathlib, sys
+ import diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings
+ for module in (diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings):
+ assert pathlib.Path(module.__file__).is_relative_to(sys.prefix), module.__file__
+ '
+ ../fresh/bin/python -m pytest -p no:cacheprovider -q --require-imports tests/test_pytorch_snapshot.py
diff --git a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
index 48752326f..fa8f81034 100644
--- a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
+++ b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
@@ -179,8 +179,14 @@ read.
The index holds rows, key maps and relations. It holds no anchors, labels or
fitted encoders, so several contracts over the same tables can share one index.
-Fitting keeps the column statistics of encoders over stored columns in
-`col_stats/`, so it needs the index directory to be writable. Each encoder
+By default, fitting keeps column statistics of encoders over stored columns in
+the index's `col_stats/`, so that directory must be writable. For the ordinary
+`DiskGraphEngine(index=...)` constructor, omitting `work_dir` or explicitly setting
+it to the index path uses this same layout and reuses the same caches. Paths that
+resolve to the index, including symlinks, are equivalent. Other paths inside the
+index are rejected. External work directories and their writable roots must be
+disjoint from the base, including after resolving symlinks. Supply an external
+`work_dir` to put newly computed statistics and task state there instead. Each encoder
cutoff has one directory there, with a `
.col_stats.json` file per table.
A later fit at the same cutoff computes only the columns and statistics those
files lack, and adds them. Ingest computes the statistics for the cutoffs in
@@ -195,6 +201,74 @@ it was built. With
`ingest="never"`, `prepare` requires an index that already exists, which is
what you usually want in a scoring job.
+### Open an existing snapshot or index
+
+`DiskGraphEngine.from_snapshot(snapshot, work_dir=...)` consumes an existing
+`kumo.engine.GraphSnapshot`. Use the engine's existing snapshot loader to
+restore an exported snapshot first. `DiskGraphEngine.from_index(index_path,
+work_dir=...)` opens the native index directory directly, including a
+PyTorch-produced index. Neither factory exports sources or ingests the base.
+
+
+```python
+engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir="/scratch/loader")
+contract = api.DataLoaderSpec(
+ schema=engine.schema,
+ sample=sample_spec,
+ batch=batch_spec,
+)
+with api.prepare(contract=contract, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ for batch in task.batches(output_format="columnar"):
+ graph = batch.inputs["graph"]
+```
+
+`SampleSpec` and `BatchSpec` retain their usual meaning. `engine.schema` derives
+stored types, keys, clocks and relationships from the existing native config
+and metadata. PyTorch-produced indexes retain their logical relation aliases;
+engine/native indexes initially use native FK names. A caller can provide a
+compatible schema projection and relation aliases: binding checks endpoints,
+FK columns and direction against the actual index. `prepare` rejects a Dataset
+alongside an existing-index engine. Anchors still select entity/time requests;
+neighborhoods are sampled during iteration.
+
+The base may be read-only. Task extensions, scalar-materialization scratch,
+new statistics, tokenizers and token-length caches go under the external work
+directory; existing immutable base caches can be read in place. Closing a task
+cleans up its temporary files. Derived caches remain for reuse. No new base
+metadata file is written. Fitted-state statistics provenance records absolute paths
+in external mode (including reused base files); these are diagnostic references,
+not inputs used when restoring fitted encoders. They can refer to the old location
+after relocation. External statistics share one directory per artifact and cutoff;
+the native writer locks, merges existing keys, and atomically replaces each file.
+Concurrent requests may duplicate computation but do not discard published keys.
+External storage requires a DiskGraph build supporting `cutoff_col_stats(output_dir=...)`;
+preparation checks that capability because older and newer development builds can
+share version 0.2.0. Install DiskGraph from the matching repository revision.
+Existing snapshot identity or PyTorch producer fingerprints survive relocation. A bare completed native index without producer
+identity is hashed in full on each opening, which can be expensive for large artifacts.
+
+Opening requires the resolved `diskgraph_config.json` and the producer's
+completion metadata. Engine snapshots use the native inventory/integrity check;
+PyTorch indexes use their existing manifest, published after ingestion. Older
+artifacts missing resolved config are refused. Native edge-table layouts,
+end-time validity and unsupported stored types (including lists) are refused
+explicitly. Snapshot adoption does not recover relations omitted during ingestion
+or enable additional sampling strategies.
+
+For **source ingestion/reuse** after connector restaging, pass
+`DiskGraphEngine(index=..., work_dir=..., source_versions=...)`. Map every table
+to `{"source": "connector/source-coordinate", "version": "connector-version"}`.
+Equal trusted identities/versions and schema permit reuse across export paths;
+a changed identity/version or schema refuses reuse. Without trustworthy versions,
+keep the existing file-validation mode. This opt-in trusts the caller to change
+the version whenever source data changes; reuse does not check Parquet contents,
+file sizes or mtimes. Reusing a stale version token therefore reuses the old index.
+Physical file checks still detect source changes during initial ingestion.
+Opening an already materialized index
+never checks whether its sources now have a newer version; refreshing sources
+remains an explicit producer operation.
+
`output_format="model"` lets the DiskGraph engine build the batch's tensors
column by column, without building a Python record per event. It refuses raw types
that a model tensor can't hold (strings, dates, decimals) before fetching any
diff --git a/packages/experimental/kumo-pytorch/docs/api/README.md b/packages/experimental/kumo-pytorch/docs/api/README.md
index 9b54cb972..a9756b849 100644
--- a/packages/experimental/kumo-pytorch/docs/api/README.md
+++ b/packages/experimental/kumo-pytorch/docs/api/README.md
@@ -108,7 +108,7 @@ The [guide](../README.md) explains the concepts in order. The behavior spec, wit
| [`InputExposureWarning`](targets.md#inputexposurewarning) | `kumo.pytorch.api` | A masked value is also selected by an unmasked named input. |
| [`DataLoaderSpec`](lifecycle.md#dataloaderspec) | `kumo.pytorch.api` | Schema, sampled values, and explicitly encoded model outputs. |
| [`check`](lifecycle.md#check) | `kumo.pytorch.api` | Return everything wrong with this combination, without raising. |
-| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind the dataset through the engine. |
+| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind sources or an existing index through the engine. |
| [`preview`](lifecycle.md#preview) | `kumo.pytorch.api` | Build up to eight raw samples and one model-facing batch. |
| [`Plan`](lifecycle.md#plan) | `kumo.pytorch.api` | A contract bound to a dataset through an engine, before any anchors. |
| [`Task`](lifecycle.md#task) | `kumo.pytorch.api` | Anchors bound to a plan: what is iterated, served from, and written out. |
diff --git a/packages/experimental/kumo-pytorch/docs/api/engines.md b/packages/experimental/kumo-pytorch/docs/api/engines.md
index 60984b721..1ea55e042 100644
--- a/packages/experimental/kumo-pytorch/docs/api/engines.md
+++ b/packages/experimental/kumo-pytorch/docs/api/engines.md
@@ -299,6 +299,8 @@ class DiskGraphEngine(
num_shards: int = 1,
ingest: Literal['if_missing', 'never'] = 'if_missing',
encoder_cutoffs: Sequence[str | int | datetime | date | None] = (),
+ work_dir: str | Path | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
)
```
@@ -330,6 +332,54 @@ step, before any data is read.
- `num_shards`: index shards.
- `ingest`: `"if_missing"` builds the index when absent; `"never"` requires an existing one. A mismatching manifest is an error, never permission to reuse.
- `encoder_cutoffs`: `encoder_cutoff` values whose column statistics an ingest computes.
+- `work_dir`: task/cache directory. Omit it or pass the index path (including an alias resolving to it) for the original index-local layout and cache reuse. Other paths must be outside the index; existing-index factories require an external directory.
+- `source_versions`: optional table-to-identity mapping for source ingestion/reuse. Each entry has nonempty `source` (connector plus source identity) and `version` strings. These trusted connector versions replace file path/size/mtime identity; missing versions are never treated as equal. Omit for legacy file validation. The caller must change the version whenever source data changes: reuse does not compare Parquet contents, file sizes or mtimes in this opt-in mode.
+
+#### `DiskGraphEngine.from_index`
+
+*classmethod*
+
+```python
+DiskGraphEngine.from_index(
+ index: str | Path,
+ *,
+ work_dir: str | Path,
+ num_threads: int | None = None,
+) -> DiskGraphEngine
+```
+
+Open a complete native index without its original sources or any base writes.
+
+Read schema/relations from existing metadata. Older indexes without the
+resolved native config are refused. Native edge tables and non-scalar
+stored columns are not supported. Bare indexes without producer identity
+are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan.
+`work_dir` holds task extensions and derived caches outside the base.
+
+#### `DiskGraphEngine.from_snapshot`
+
+*classmethod*
+
+```python
+DiskGraphEngine.from_snapshot(
+ snapshot: Any,
+ *,
+ work_dir: str | Path,
+ num_threads: int | None = None,
+) -> DiskGraphEngine
+```
+
+Bind a kumo.engine GraphSnapshot using its existing index and identity.
+
+Uses public `index_path` and `id` fields; importing kumo.engine is
+unnecessary. Load exported snapshots through the existing engine loader.
+No source export, base ingestion or additional metadata file is needed.
+
+#### `DiskGraphEngine.schema`
+
+*property*
+
+Resolved stored schema for a from_index/from_snapshot engine.
#### `DiskGraphEngine.check_contract`
diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
index 85aefeb1d..264eabe86 100644
--- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
+++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
@@ -144,8 +144,8 @@ A readable multi-line summary of the contract.
```python
def check(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
@@ -169,14 +169,18 @@ The whole contract must still be well formed.
```python
def prepare(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
inference: bool = False,
) -> Plan
```
-Validate, then bind the dataset through the engine.
+Validate, then bind sources or an existing index through the engine.
+
+Omit `dataset` only for a DiskGraphEngine created with `from_index` or
+`from_snapshot`. Its stored schema supplies omitted types; it never reads
+the original sources or rebuilds the base. `engine` is always required.
Every check that does not need data happens first, so an unexecutable
combination is refused before a row is read. Anchors, and everything fitted
@@ -221,7 +225,7 @@ Called as `preview(task, requests)` or
```python
class Plan(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
source: SampleSource,
*,
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
new file mode 100644
index 000000000..b8c34a8a2
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Storage policy for index-local and external consumer state."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def normalize_work_dir(
+ index: str | Path, work_dir: str | Path | None, *, external_only: bool = False
+) -> Path | None:
+ """Use the legacy layout for an omitted or explicitly equal index path."""
+ base = Path(index).resolve()
+ work = None if work_dir is None else Path(work_dir).resolve()
+ if not external_only and (work is None or work == base):
+ return None
+ if work is None:
+ raise ValueError("work_dir must be outside the base index")
+ # Validate every writable root before creating any of them, including symlinks.
+ for destination in (work, work / "tasks", work / "train_tables", work / "cache"):
+ resolved = destination.resolve()
+ if resolved.is_relative_to(base) or base.is_relative_to(resolved):
+ raise ValueError(
+ "work_dir and its writable roots must be outside and disjoint from the base index"
+ )
+ return work
+
+
+class ColumnCache:
+ """Read immutable base caches and publish missing entries to the writable root."""
+
+ def __init__(self, index: str | Path, root: str | Path | None = None):
+ self.index = Path(index)
+ self.root = self.index if root is None else Path(root)
+ self.external = self.root.resolve() != self.index.resolve()
+ if not self.external:
+ self.root = self.index
+
+ @classmethod
+ def for_work_dir(cls, index: str | Path, work_dir: Path | None, identity: str):
+ root = (
+ None
+ if work_dir is None
+ else work_dir / "cache" / hashlib.sha256(identity.encode()).hexdigest()
+ )
+ return cls(index, root)
+
+ def destination(self, name: str) -> Path:
+ return self.root / "col_stats" / name
+
+ def candidates(self, name: str, *, base_name: str | None = None) -> tuple[Path, ...]:
+ base = self.index / "col_stats" / (name if base_name is None else base_name)
+ return tuple(dict.fromkeys((base, self.destination(name))))
+
+ def resolve(self, name: str) -> Path:
+ return next(
+ (path for path in self.candidates(name) if path.is_file()), self.destination(name)
+ )
+
+ def reference(self, path: Path) -> str:
+ return str(path if self.external else path.relative_to(self.index / "col_stats"))
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
index d1cf37703..e219759e7 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
@@ -20,6 +20,7 @@
import numpy as np
+from kumo.pytorch._storage import ColumnCache
from kumo.pytorch.contract.batch import (
Composite,
EntitySource,
@@ -243,7 +244,7 @@ def token_lengths_file(
def read_token_lengths(
- index: str | Path, entry: Mapping[str, Any]
+ index: str | Path, entry: Mapping[str, Any], *, cache_root: str | Path | None = None
) -> tuple[int, np.ndarray | None]:
"""A table's per-event constant and its read-only per-node counts, indexed by node ID.
@@ -254,7 +255,7 @@ def read_token_lengths(
if entry["file"] is None:
return entry["constant"], None
- path = Path(index, "col_stats", entry["file"])
+ path = ColumnCache(index, cache_root).resolve(entry["file"])
if not path.stat().st_size:
return entry["constant"], np.frombuffer(b"", dtype=" ColumnStats:
- """The requested statistics at a cutoff, first computing into the index those it lacks."""
+ """Read stored statistics, computing missing values into the selected cache root."""
import diskgraph
- directory = Path(index, "col_stats", _cutoff_dir(ticks, unit))
- paths = {
- table: directory / f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests
- }
+ cache = ColumnCache(index, cache_root)
+ cutoff = _cutoff_dir(ticks, unit)
+ # Native publication locks, merges existing keys, and atomically replaces files.
+ # Sharing a cutoff directory lets overlapping requests reuse prior statistics.
+ relative = cutoff
+ paths = {table: f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests}
+
+ files = set()
def read() -> dict:
- return {
- table: json.loads(path.read_text())[table]
- for table, path in paths.items()
- if path.is_file()
- }
+ found = {}
+ for table, name in paths.items():
+ for candidate in cache.candidates(f"{relative}/{name}", base_name=f"{cutoff}/{name}"):
+ if candidate.is_file():
+ files.add(candidate)
+ data = json.loads(candidate.read_text())[table]
+ for column, values in data.items():
+ found.setdefault(table, {}).setdefault(column, {}).update(values)
+ return found
found = read()
missing = [
- (table, column, stype)
- for table, column, stype in requests
- if _STAT_KEYS[stype] not in found.get(table, {}).get(column, {})
+ item
+ for item in requests
+ if _STAT_KEYS[item[2]] not in found.get(item[0], {}).get(item[1], {})
]
if missing:
request = stats_request(missing, [ticks], unit)
- diskgraph.cutoff_col_stats(str(index), json.dumps(request), num_threads=num_threads)
+ request["dirs"] = [relative]
+ options = {"output_dir": str(cache.root)} if cache.external else {}
+ diskgraph.cutoff_col_stats(
+ str(index), json.dumps(request), num_threads=num_threads, **options
+ )
found = read()
return ColumnStats(
{item: found[item[0]][item[1]] for item in requests},
{"value": ticks, "unit": unit},
- tuple(f"{directory.name}/{path.name}" for _, path in sorted(paths.items())),
+ tuple(cache.reference(path) for path in sorted(files)),
)
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
index 87eab16dd..b81d83a4c 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
@@ -5,13 +5,14 @@
"""The engine interface: what an engine must provide, and how a caller picks one.
A :class:`~kumo.pytorch.dataset.Dataset` says where the data is;
-an :class:`Engine` says what reads it. Both are supplied by the caller, and
-neither names the other.
+an :class:`Engine` says what reads it. Source ingestion supplies both.
+An existing-index ``DiskGraphEngine`` supplies the data binding itself, so
+``prepare`` and ``check`` accept it without a ``Dataset``.
An engine is a factory, not a holder of data: :meth:`Engine.open` binds a
dataset and returns a :class:`SampleSource`. Construction-time configuration
(artifact directories, index paths) belongs to the engine; data binding belongs
-to the dataset.
+to the dataset or the explicitly adopted index.
Distributed execution is expressed by ``rank``/``world_size`` on
``Task.batches``.
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
index 1d7b7a10c..7456c398f 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
@@ -8,6 +8,7 @@
import datetime as dt
import hashlib
+import inspect
import json
import os
import time
@@ -21,6 +22,7 @@
import numpy as np
import torch
+from kumo.pytorch._storage import ColumnCache, normalize_work_dir
from kumo.pytorch.anchors import AnchorTable
from kumo.pytorch.column_stats import (
ColumnStats,
@@ -73,6 +75,7 @@
from kumo.pytorch.engine import AnchorTask, GraphRoot, SampleSource, register_engine
from kumo.pytorch.engines.parquet_anchors import ParquetAnchors, check_parquet_anchors
from kumo.pytorch.engines.unsupported import DISKGRAPH_UNSUPPORTED, Unsupported
+from kumo.pytorch.index_binding import open_index
from kumo.pytorch.materialization import _DiskGraphIndex, ensure_diskgraph_index
from kumo.pytorch.spec import DataLoaderSpec, ValidationIssue
@@ -214,6 +217,16 @@ class DiskGraphEngine:
requires an existing one. A mismatching manifest is an error, never
permission to reuse.
:param encoder_cutoffs: ``encoder_cutoff`` values whose column statistics an ingest computes.
+ :param work_dir: task/cache directory. Omit it or pass the index path (including
+ an alias resolving to it) for the original index-local layout and cache reuse.
+ Other paths must be outside the index; existing-index factories require an
+ external directory.
+ :param source_versions: optional table-to-identity mapping for source ingestion/reuse.
+ Each entry has nonempty ``source`` (connector plus source identity) and ``version``
+ strings. These trusted connector versions replace file path/size/mtime identity;
+ missing versions are never treated as equal. Omit for legacy file validation.
+ The caller must change the version whenever source data changes: reuse does
+ not compare Parquet contents, file sizes or mtimes in this opt-in mode.
"""
name = "diskgraph"
@@ -228,12 +241,21 @@ def __init__(
num_shards: int = 1,
ingest: Literal["if_missing", "never"] = "if_missing",
encoder_cutoffs: Sequence[str | int | dt.datetime | dt.date | None] = (),
+ work_dir: str | Path | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
):
if ingest not in {"if_missing", "never"}:
raise ValueError("ingest must be 'if_missing' or 'never'")
if isinstance(encoder_cutoffs, str):
raise TypeError("encoder_cutoffs is a list of cutoffs; wrap a single one in a list")
self.index = Path(index)
+ self.work_dir = normalize_work_dir(self.index, work_dir)
+ self.source_versions = (
+ None
+ if source_versions is None
+ else {name: dict(entry) for name, entry in source_versions.items()}
+ )
+ self._binding = None
self.num_threads = num_threads
self.num_shards = num_shards
self.ingest = ingest
@@ -241,6 +263,68 @@ def __init__(
dict.fromkeys(cutoff_nanos(value) for value in encoder_cutoffs)
)
+ @classmethod
+ def from_index(
+ cls, index: str | Path, *, work_dir: str | Path, num_threads: int | None = None
+ ) -> DiskGraphEngine:
+ """Open a complete native index without its original sources or any base writes.
+
+ Read schema/relations from existing metadata. Older indexes without the
+ resolved native config are refused. Native edge tables and non-scalar
+ stored columns are not supported. Bare indexes without producer identity
+ are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan.
+ ``work_dir`` holds task extensions and derived caches outside the base.
+ """
+ work_dir = normalize_work_dir(index, work_dir, external_only=True)
+ result = cls(index=index, ingest="never", work_dir=work_dir, num_threads=num_threads)
+ result._binding = open_index(index)
+ return result
+
+ @classmethod
+ def from_snapshot(
+ cls, snapshot: Any, *, work_dir: str | Path, num_threads: int | None = None
+ ) -> DiskGraphEngine:
+ """Bind a kumo.engine GraphSnapshot using its existing index and identity.
+
+ Uses public ``index_path`` and ``id`` fields; importing kumo.engine is
+ unnecessary. Load exported snapshots through the existing engine loader.
+ No source export, base ingestion or additional metadata file is needed.
+ """
+ work_dir = normalize_work_dir(snapshot.index_path, work_dir, external_only=True)
+ result = cls(
+ index=snapshot.index_path, ingest="never", work_dir=work_dir, num_threads=num_threads
+ )
+ result._binding = open_index(snapshot.index_path, snapshot_id=snapshot.id)
+ return result
+
+ @property
+ def schema(self) -> DataSchema:
+ """Resolved stored schema for a from_index/from_snapshot engine."""
+ if self._binding is None:
+ raise ValueError("schema is available on from_index/from_snapshot engines")
+ return self._binding.schema
+
+ def _resolve_index_schema(self, schema: DataSchema) -> DataSchema:
+ if self._binding is None:
+ raise ValueError("dataset is required unless the engine binds an existing index")
+ return self._binding.resolve(schema)[0]
+
+ def _check_binding(self, dataset: Dataset | None, schema: DataSchema):
+ if self._binding is not None and dataset is not None:
+ return (
+ ValidationIssue(
+ "dataset",
+ "an existing-index engine does not accept Dataset",
+ "omit dataset; the index supplies the table binding",
+ ),
+ )
+ if dataset is None:
+ try:
+ self._resolve_index_schema(schema)
+ except ValueError as error:
+ return (ValidationIssue("schema", str(error)),)
+ return ()
+
def check_contract(self, contract: DataLoaderSpec) -> tuple[ValidationIssue, ...]:
"""Refuse the entries of `unsupported` the contract uses, before any data is read."""
issues = _sample_refusals(contract.sample)
@@ -394,11 +478,38 @@ def check_anchors(table: AnchorTable) -> tuple[ValidationIssue, ...]:
def open(
self,
- dataset: Dataset,
+ dataset: Dataset | None,
schema: DataSchema,
*,
column_stats: Sequence[StatRequest] = (),
) -> SampleSource:
+ if self.work_dir is not None:
+ import diskgraph
+
+ try:
+ inspect.signature(diskgraph.cutoff_col_stats).bind(
+ str(self.index), "{}", output_dir=str(self.work_dir)
+ )
+ except (TypeError, ValueError) as error:
+ raise RuntimeError(
+ "External work_dir requires a DiskGraph build with "
+ "cutoff_col_stats(output_dir=...) support; rebuild/install DiskGraph "
+ "from the matching kumo-engine revision"
+ ) from error
+ if self._binding is not None:
+ if dataset is not None:
+ raise ValueError("an existing-index engine does not accept Dataset")
+ self._binding.validate_files()
+ schema, artifact = self._binding.resolve(schema)
+ return DiskGraphSource(
+ artifact,
+ schema,
+ num_threads=self.num_threads,
+ work_dir=self.work_dir,
+ identity=self._binding.identity,
+ )
+ if dataset is None:
+ raise ValueError("dataset is required for source ingestion")
ingests = bool(column_stats and self.encoder_cutoffs) and not self.index.exists()
artifact = ensure_diskgraph_index(
schema,
@@ -408,11 +519,13 @@ def open(
num_threads=self.num_threads,
if_missing=self.ingest == "if_missing",
column_stats=(column_stats, self.encoder_cutoffs) if ingests else None,
+ source_versions=self.source_versions,
)
return DiskGraphSource(
artifact,
schema,
num_threads=self.num_threads,
+ work_dir=self.work_dir,
)
def materialize_anchors(self, table: AnchorTable) -> AnchorTask:
@@ -430,6 +543,8 @@ def __init__(
schema: DataSchema,
*,
num_threads: int | None,
+ work_dir: Path | None = None,
+ identity: str | None = None,
) -> None:
try:
import diskgraph
@@ -445,6 +560,16 @@ def __init__(
self.schema = schema
self._index_path = artifact.path
+ self.artifact_identity = identity
+ self._work_dir = Path(artifact.path) if work_dir is None else work_dir
+ cache_id = (
+ identity
+ or hashlib.sha256(
+ json.dumps(dict(artifact.manifest), sort_keys=True).encode()
+ ).hexdigest()
+ )
+ self._column_cache = ColumnCache.for_work_dir(artifact.path, work_dir, cache_id)
+ self._cache_root = self._column_cache.root
metadata = json.loads(Path(artifact.path, "index_metadata.json").read_text())
self._clock = metadata.get("timestamp_unit", "seconds")
self._num_threads = num_threads
@@ -1123,7 +1248,14 @@ def column_stats(self, requests: Sequence[StatRequest], cutoff: int | None) -> C
if cutoff is None
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
- return read_column_stats(self._index_path, requests, ticks, self._clock, self._num_threads)
+ return read_column_stats(
+ self._index_path,
+ requests,
+ ticks,
+ self._clock,
+ self._num_threads,
+ cache_root=self._cache_root,
+ )
def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[str, str]:
ticks = (
@@ -1132,7 +1264,7 @@ def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = encoder.tokenizer_file(ticks, self._clock)
- path = Path(self._index_path, "col_stats", name)
+ path = self._column_cache.resolve(name)
if path.is_file():
try:
document = json.loads(path.read_text())
@@ -1181,17 +1313,19 @@ def token_lengths(
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = token_lengths_file(table, columns, required, ticks, self._clock)
- path = Path(self._index_path, "col_stats", name)
+ path = self._column_cache.resolve(name)
if not path.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}")
self._diskgraph.token_lengths(
self._index_path,
table,
list(columns),
- str(path),
+ str(temporary),
required=list(required),
num_threads=self._num_threads,
)
+ temporary.replace(path)
return name
def fit_graph_fields(
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
index 32bb6ee96..cbb892d94 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
@@ -74,8 +74,8 @@ def __init__(self, source: Any, sample: Any, table: AnchorTable, *, directory=No
self._source, self._sample, self._table = source, sample, table
self._feature_types = sample_value_types(source.schema, sample)
self._pa, self._pc = pa, pc
- parent = Path(source._index_path)
- (parent / "tasks").mkdir(exist_ok=True)
+ parent = Path(source._work_dir)
+ (parent / "tasks").mkdir(parents=True, exist_ok=True)
(parent / "train_tables").mkdir(exist_ok=True)
self._temporary = directory or tempfile.TemporaryDirectory(
prefix="task-", dir=parent / "tasks"
@@ -470,7 +470,9 @@ def bind_token_lengths(self, assembler, contract, statistics):
raise ValueError(
"event token costs do not match the designated input's fitted recipe"
)
- fixed, values = read_token_lengths(self._source._index_path, entry)
+ fixed, values = read_token_lengths(
+ self._source._index_path, entry, cache_root=self._source._cache_root
+ )
if values is not None:
self._token_maps.append(values)
self._event_costs[table] = dict(
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py
new file mode 100644
index 000000000..a830e0073
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py
@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read existing DiskGraph artifacts without their original table sources."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from kumo.pytorch.contract.dtypes import DType
+from kumo.pytorch.contract.schema import NON_TEMPORAL, DataSchema, RelationSpec, TableSpec
+from kumo.pytorch.materialization import _DiskGraphIndex, _fingerprint, _foreign_key
+
+_CLOCKS = {"seconds": "s", "milliseconds": "ms", "microseconds": "us", "nanoseconds": "ns"}
+
+
+def _read(root: Path, name: str) -> dict:
+ try:
+ value = json.loads((root / name).read_text())
+ except (OSError, ValueError) as error:
+ raise ValueError(f"cannot open DiskGraph metadata {root / name}: {error}") from error
+ if not isinstance(value, dict):
+ raise ValueError(f"DiskGraph metadata {root / name} must be an object")
+ return value
+
+
+def _dtype(column: dict, clock: str, table: str) -> DType:
+ name = column["ingested_dtype"]
+ name = {"date": "date32", "utf8": "string"}.get(name, name.replace("u_int", "uint"))
+ try:
+ if name == "timestamp":
+ return DType(name, unit=_CLOCKS[clock], timezone="UTC")
+ if name.startswith("decimal"):
+ return DType(name, precision=column["precision"], scale=column["scale"])
+ if column.get("semantic_type") == "multicategorical":
+ raise ValueError("multicategorical storage requires list support")
+ return DType(name)
+ except (KeyError, ValueError) as error:
+ raise ValueError(
+ f"unsupported stored type for {table}.{column['name']}: {column}"
+ ) from error
+
+
+@dataclass(frozen=True)
+class IndexBinding:
+ artifact: _DiskGraphIndex
+ schema: DataSchema
+ identity: str
+ metadata_fingerprint: str
+ physical_relations: tuple[tuple[RelationSpec, dict], ...]
+
+ def resolve(self, schema: DataSchema) -> tuple[DataSchema, _DiskGraphIndex]:
+ """Validate a projection and map consumer relation aliases to physical edges."""
+ stored = {table.name: table for table in self.schema.tables}
+ tables = []
+ for table in schema.tables:
+ actual = stored.get(table.name)
+ if actual is None:
+ raise ValueError(f"index has no node table {table.name!r}")
+ if (table.primary_key, table.time_column) != (actual.primary_key, actual.time_column):
+ raise ValueError(f"index keys/clock differ for table {table.name!r}")
+ columns = actual.columns if table.columns is None else table.columns
+ resolved = {}
+ for name, dtype in columns.items():
+ if name not in actual.columns:
+ raise ValueError(f"index has no column {table.name}.{name}")
+ expected = actual.columns[name]
+ if dtype is not None and dtype != expected:
+ raise ValueError(
+ f"index stored type differs for {table.name}.{name}: {expected}"
+ )
+ resolved[name] = expected
+ tables.append(replace(table, columns=resolved))
+ mapping = {}
+ definitions = {table.name: table for table in tables}
+ for relation in schema.relations:
+ try:
+ child, fk, parent, forward = _foreign_key(relation, definitions)
+ except KeyError as error:
+ raise ValueError(
+ f"relation {relation.name!r} refers to an undeclared table"
+ ) from error
+ matches = [
+ native
+ for physical, native in self.physical_relations
+ if (physical.left_table, physical.left_key, physical.right_table)
+ == (child, fk, parent)
+ ]
+ if len(matches) != 1:
+ raise ValueError(f"index has no unambiguous stored relation for {relation.name!r}")
+ native = matches[0]
+ mapping[relation.name] = {
+ "left_table": relation.left_table,
+ "right_table": relation.right_table,
+ "forward": native["forward" if forward else "reverse"],
+ "reverse": native["reverse" if forward else "forward"],
+ }
+ return DataSchema(tuple(tables), schema.relations), _DiskGraphIndex(
+ self.artifact.path, {"relation_mapping": mapping}
+ )
+
+ def validate_files(self) -> None:
+ root = Path(self.artifact.path)
+ if _metadata_fingerprint(root) != self.metadata_fingerprint:
+ raise ValueError("DiskGraph index metadata changed after binding; open it again")
+ _check_complete(root)
+
+
+def _check_complete(root: Path) -> None:
+ import diskgraph
+
+ if (root / "_MANIFEST.json").is_file():
+ diskgraph.verify_snapshot_integrity(str(root))
+ elif (root / "index_manifest.json").is_file():
+ # PyTorch publishes this manifest by atomic rename after native ingest;
+ # that producer does not request engine _MANIFEST/_SUCCESS markers.
+ _DiskGraphIndex.open(root)
+ else:
+ raise ValueError(f"incomplete DiskGraph index: {root} has no completion manifest")
+
+
+def _metadata_fingerprint(root: Path) -> str:
+ return _fingerprint(
+ {
+ name: hashlib.sha256((root / name).read_bytes()).hexdigest()
+ for name in (
+ "diskgraph_config.json",
+ "index_metadata.json",
+ "_MANIFEST.json",
+ "index_manifest.json",
+ "_provenance.json",
+ )
+ if (root / name).is_file()
+ }
+ )
+
+
+def open_index(path: str | Path, *, snapshot_id: str | None = None) -> IndexBinding:
+ """Build an in-memory binding; write nothing and never follow stored source paths."""
+ root = Path(path).resolve()
+ if not (root / "diskgraph_config.json").is_file():
+ raise ValueError(
+ f"{root} has no diskgraph_config.json; automatic opening of this legacy "
+ "index is unsupported (no schema is inferred from unavailable sources)"
+ )
+ _check_complete(root)
+ config, metadata = _read(root, "diskgraph_config.json"), _read(root, "index_metadata.json")
+ clock = metadata.get("timestamp_unit", "seconds")
+ if clock not in _CLOCKS or config.get("timestamp_unit", "seconds") != clock:
+ raise ValueError(f"unsupported or inconsistent DiskGraph clock: {clock!r}")
+ nodes = {node["name"]: node for node in metadata["node_types"]}
+ tables = []
+ for table in config["tables"]:
+ name = table["name"]
+ if table.get("edge_table") is not None:
+ raise ValueError(f"native edge table {name!r} is not supported by the PyTorch binding")
+ node = nodes.get(name)
+ if node is None:
+ raise ValueError(f"config table {name!r} is absent from native node metadata")
+ key, time = table.get("primary_key"), table.get("create_time")
+ if key != (node.get("primary_key_map") or {}).get("column"):
+ raise ValueError(f"inconsistent native primary key for {name!r}")
+ if time != node.get("create_time_col") or table.get("end_time"):
+ raise ValueError(f"inconsistent or unsupported native clock/end_time for {name!r}")
+ tables.append(
+ TableSpec(
+ name,
+ key,
+ {col["name"]: _dtype(col, clock, name) for col in table["columns"]},
+ time_column=time or NON_TEMPORAL,
+ )
+ )
+ if set(nodes) != {table.name for table in tables}:
+ raise ValueError("native node types differ from the resolved config")
+ by_name = {table.name: table for table in tables}
+ edges = {edge["name"]: edge for edge in metadata["edge_relations"]}
+ physical = []
+ for table in config["tables"]:
+ for fk in table.get("fkeys", []):
+ child, column, parent = table["name"], fk["src_column"], fk["dst_table"]
+ if parent not in by_name or column not in by_name[child].columns:
+ raise ValueError(f"invalid native foreign key {child}.{column} -> {parent}")
+ pk = by_name[parent].primary_key
+ # Native FK naming is defined by ingest; verify the name AND endpoints/pair.
+ name = f"{child}_{column}_to_{parent}_{pk}"
+ edge = edges.get(name)
+ reverse = edges.get((edge or {}).get("reverse_of"))
+ if (
+ edge is None
+ or reverse is None
+ or (edge["src_type"], edge["dst_type"]) != (child, parent)
+ or (reverse["src_type"], reverse["dst_type"]) != (parent, child)
+ or reverse.get("reverse_of") != name
+ ):
+ raise ValueError(f"missing or inconsistent native FK edge pair for {name!r}")
+ for direction in (edge, reverse):
+ target = by_name[direction["dst_type"]]
+ expected = (
+ None
+ if target.time_column == NON_TEMPORAL
+ else (target.name, target.time_column)
+ )
+ timestamp = (direction.get("dest_sort_key") or {}).get("timestamp")
+ actual = None if timestamp is None else (timestamp["table"], timestamp["column"])
+ if actual != expected:
+ raise ValueError(f"unsupported stored edge clock for {direction['name']!r}")
+ physical.append(
+ (
+ RelationSpec(name, child, column, parent, pk),
+ {
+ "left_table": child,
+ "right_table": parent,
+ "forward": name,
+ "reverse": reverse["name"],
+ },
+ )
+ )
+ legacy = _DiskGraphIndex.open(root).manifest if (root / "index_manifest.json").is_file() else {}
+ relations = []
+ for physical_relation, mapping in physical:
+ aliases = [
+ (alias, entry)
+ for alias, entry in legacy.get("relation_mapping", {}).items()
+ if entry["forward"] in (mapping["forward"], mapping["reverse"])
+ ]
+ if not aliases:
+ relations.append(physical_relation)
+ for alias, entry in aliases:
+ if entry["forward"] == mapping["forward"]:
+ relations.append(replace(physical_relation, name=alias))
+ else:
+ relations.append(
+ RelationSpec(
+ alias,
+ physical_relation.right_table,
+ physical_relation.right_key,
+ physical_relation.left_table,
+ physical_relation.left_key,
+ )
+ )
+ schema = DataSchema(tuple(tables), tuple(relations))
+ provenance = _read(root, "_provenance.json") if (root / "_provenance.json").is_file() else {}
+ recorded_id = provenance.get("id")
+ if snapshot_id and recorded_id and snapshot_id != recorded_id:
+ raise ValueError("snapshot identity differs from persisted provenance")
+ identity = snapshot_id or recorded_id
+ if not identity and legacy:
+ identity = _fingerprint(
+ {
+ key: legacy[key]
+ for key in ("dataset_fingerprint", "schema_fingerprint", "config_fingerprint")
+ }
+ )
+ if not identity:
+ # Bare native indexes have no guaranteed producer identity. Hash the existing
+ # artifact once, rather than equating same-sized files or inventing metadata.
+ digest = hashlib.sha256()
+ for entry in sorted(_read(root, "_MANIFEST.json")["files"], key=lambda e: e["rel_path"]):
+ file = root / entry["rel_path"]
+ if not file.resolve().is_relative_to(root):
+ raise ValueError("native manifest contains a path outside the index")
+ digest.update(entry["rel_path"].encode())
+ with file.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ identity = digest.hexdigest()
+ binding = IndexBinding(
+ _DiskGraphIndex(str(root), {}),
+ schema,
+ str(identity),
+ _metadata_fingerprint(root),
+ tuple(physical),
+ )
+ _, artifact = binding.resolve(schema)
+ return replace(binding, artifact=artifact)
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
index fc698a929..aa79cf0ef 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
@@ -56,9 +56,9 @@ def open(cls, path: str | Path) -> _DiskGraphIndex:
raise ValueError(f"{root} is missing DiskGraph index_metadata.json")
return cls(str(root), manifest)
- def validate_binding(self, schema: DataSchema, dataset: Dataset) -> None:
+ def validate_binding(self, schema: DataSchema, dataset: Dataset, source_versions=None) -> None:
expected_schema = _fingerprint(_schema_payload(schema))
- expected_dataset = _fingerprint(_dataset_payload(dataset))
+ expected_dataset = _fingerprint(_dataset_payload(dataset, source_versions))
if self.manifest.get("schema_fingerprint") != expected_schema:
raise ValueError("DiskGraph index was materialized from a different schema")
if self.manifest.get("dataset_fingerprint") != expected_dataset:
@@ -75,6 +75,7 @@ def ensure_diskgraph_index(
dataset_name: str = "default",
if_missing: bool = True,
column_stats: tuple[Sequence[StatRequest], Sequence[int | None]] | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
) -> _DiskGraphIndex:
"""Reuse or build the base index while opening a DiskGraph engine.
@@ -92,10 +93,16 @@ def ensure_diskgraph_index(
if not dataset_name:
raise ValueError("dataset_name must be non-empty")
+ for name, source in dataset.tables.items():
+ if isinstance(source, Parquet) and source.columns:
+ raise DISKGRAPH_UNSUPPORTED["DG-COLUMN-RENAME"].error(
+ "DiskGraph materialization requires matching physical column names; "
+ f"table {name!r} declares {dict(source.columns)!r}"
+ )
output = Path(destination).resolve()
if output.exists():
artifact = _DiskGraphIndex.open(output)
- artifact.validate_binding(schema, dataset)
+ artifact.validate_binding(schema, dataset, source_versions)
_warn_dropped_rows(output)
return artifact
if not if_missing:
@@ -103,7 +110,10 @@ def ensure_diskgraph_index(
output.parent.mkdir(parents=True, exist_ok=True)
diskgraph = _import_diskgraph()
- source_snapshot = _dataset_payload(dataset)
+ physical_snapshot = _dataset_payload(dataset)
+ source_snapshot = (
+ physical_snapshot if source_versions is None else _dataset_payload(dataset, source_versions)
+ )
config, relation_mapping, source_types = _compile_diskgraph_config(
schema,
dataset,
@@ -148,7 +158,7 @@ def ensure_diskgraph_index(
**thread_options,
)
elapsed = time.perf_counter() - started
- if _dataset_payload(dataset) != source_snapshot:
+ if _dataset_payload(dataset) != physical_snapshot:
raise ValueError("Parquet files changed while DiskGraph was ingesting them")
manifest = {
@@ -426,7 +436,20 @@ def _schema_payload(schema: DataSchema) -> dict[str, Any]:
}
-def _dataset_payload(dataset: Dataset) -> dict[str, Any]:
+def _dataset_payload(dataset: Dataset, source_versions=None) -> dict[str, Any]:
+ if source_versions is not None:
+ if set(source_versions) != set(dataset.tables):
+ raise ValueError("source_versions must identify every dataset table exactly once")
+ result = {}
+ for name, entry in source_versions.items():
+ if set(entry) != {"source", "version"} or any(
+ not isinstance(value, str) or not value for value in entry.values()
+ ):
+ raise ValueError(
+ "source_versions entries require nonempty source and version strings"
+ )
+ result[name] = dict(entry)
+ return {"source_versions": result}
payload = {}
for name, source in sorted(dataset.tables.items()):
if not isinstance(source, Parquet):
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
index 0d40d51d0..448de5184 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
@@ -69,8 +69,8 @@
def check(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
@@ -91,12 +91,14 @@ def check(
def _check(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
) -> tuple[DataLoaderSpec, tuple[ValidationIssue, ...]]:
+ if engine is None:
+ raise TypeError("engine is required")
entity_issues = _entity_key_issues(contract)
if entity_issues:
return contract, entity_issues
@@ -105,6 +107,8 @@ def _check(
try:
# Keep reporting declaration issues for a bad binding when types are
# already known, but never read metadata through a missing binding.
+ if dataset is None and not issues:
+ contract = replace(contract, schema=engine._resolve_index_schema(contract.schema))
contract = contract.resolve(dataset=None if issues else dataset)
except (ValueError, KeyError, OSError) as error:
if not issues:
@@ -163,7 +167,7 @@ class Plan:
def __init__(
self,
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
source: SampleSource,
*,
@@ -174,7 +178,9 @@ def __init__(
self.engine = engine
self.inference = inference
self._source: SampleSource | None = source
- self._dataset_fingerprint = _fingerprint(dataset)
+ self._dataset_fingerprint = getattr(source, "artifact_identity", None) or _fingerprint(
+ dataset
+ )
def materialize(
self,
@@ -272,8 +278,8 @@ def materialize(
from kumo.pytorch.aggregate_materialization import materialize_values
- parent = Path(source._index_path) / "tasks"
- parent.mkdir(exist_ok=True)
+ parent = Path(source._work_dir) / "tasks"
+ parent.mkdir(parents=True, exist_ok=True)
directory = tempfile.TemporaryDirectory(prefix="task-", dir=parent)
try:
values = materialize_values(
@@ -574,12 +580,16 @@ def reused_state(
def prepare(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
inference: bool = False,
) -> Plan:
- """Validate, then bind the dataset through the engine.
+ """Validate, then bind sources or an existing index through the engine.
+
+ Omit ``dataset`` only for a DiskGraphEngine created with ``from_index`` or
+ ``from_snapshot``. Its stored schema supplies omitted types; it never reads
+ the original sources or rebuilds the base. ``engine`` is always required.
Every check that does not need data happens first, so an unexecutable
combination is refused before a row is read. Anchors, and everything fitted
@@ -660,10 +670,17 @@ def preview(
def _binding_issues(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
) -> list[ValidationIssue]:
issues: list[ValidationIssue] = []
+ _check_binding = getattr(engine, "_check_binding", None)
+ if _check_binding is not None:
+ issues.extend(_check_binding(dataset, contract.schema))
+ if dataset is None:
+ if _check_binding is None:
+ issues.append(ValidationIssue("dataset", "this engine requires a Dataset"))
+ return issues
declared = [table.name for table in contract.schema.tables]
missing = dataset.missing_for(declared)
if missing:
diff --git a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
index 50a645713..7eb3aa953 100644
--- a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
+++ b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
@@ -86,7 +86,14 @@ class Shape:
class _Source:
"""In place of DiskGraphSource: what the engine hands the sampler."""
- def __init__(self, artifact: _DiskGraphIndex, schema: DataSchema, *, num_threads: int | None):
+ def __init__(
+ self,
+ artifact: _DiskGraphIndex,
+ schema: DataSchema,
+ *,
+ num_threads: int | None,
+ work_dir: Path | None = None,
+ ):
self.artifact, self.schema, self.num_threads = artifact, schema, num_threads
@@ -381,7 +388,11 @@ def _resize(path: Path) -> None:
@pytest.mark.parametrize("change", ("mtime", "size", "extra_file"))
def test_a_changed_dataset_is_refused_not_reused(change: str, tmp_path: Path) -> None:
- """M06: the dataset fingerprint covers each file's path, size and mtime."""
+ """M06: source ingestion without connector versions validates path/size/mtime.
+
+ Explicit existing-index adoption and version-aware restaging are covered
+ by test_existing_index; those modes do not require the original files.
+ """
with _built(tmp_path, parts=2) as (dataset, index):
before = _snapshot(index)
diff --git a/packages/experimental/kumo-pytorch/tests/test_existing_index.py b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
new file mode 100644
index 000000000..56c947f12
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
@@ -0,0 +1,697 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Source-free, read-only reuse through the public preparation lifecycle."""
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from kumo.pytorch import api
+from kumo.pytorch.column_stats import TableColumnEncoder, read_token_lengths
+
+diskgraph = pytest.importorskip("diskgraph")
+
+
+def inventory(root):
+ return {
+ str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in root.rglob("*")
+ if p.is_file()
+ }
+
+
+@contextmanager
+def read_only(root):
+ paths = [root, *root.rglob("*")]
+ modes = {p: p.stat().st_mode for p in paths}
+ for p in paths:
+ p.chmod(0o555 if p.is_dir() else 0o444)
+ try:
+ yield
+ finally:
+ for p, mode in modes.items():
+ p.chmod(mode)
+
+
+def contract(schema):
+ return api.DataLoaderSpec(
+ schema,
+ api.SampleSpec(
+ entity=api.EntitySpec("users"),
+ neighborhoods={
+ "history": api.NeighborhoodSpec(
+ (api.GraphLayer((api.GraphSample("orders", fanout=2, strategy="recent"),)),)
+ ),
+ },
+ ),
+ api.BatchSpec(
+ inputs={
+ "graph": api.Input(
+ api.NeighborhoodSource("history"),
+ api.Graph(
+ node_fields={
+ table.name: api.Fields(
+ tuple(api.Encode(name, api.Raw()) for name in table.columns)
+ )
+ for table in schema.tables
+ }
+ ),
+ None,
+ )
+ },
+ batching=api.Batching(max_samples=2),
+ ),
+ )
+
+
+@pytest.fixture
+def indexed(tmp_path):
+ source = tmp_path / "sources"
+ source.mkdir()
+ data = {
+ "users": pa.table({"id": [-2, 7], "name": ["Alice", "Bob"]}),
+ "events": pa.table(
+ {
+ "id": ["a", "b", "c", "d"],
+ "user": [-2, -2, -2, 7],
+ "reviewer": [7, 7, 7, -2],
+ "parent": pa.array([None, "a", "b", None], pa.string()),
+ "at": pa.array([10, 20, 30, 10], pa.timestamp("s")),
+ "value": [2.0, 4.0, 100.0, None],
+ "note": ["first item", "second item", "future item", None],
+ }
+ ),
+ }
+ for name, table in data.items():
+ pq.write_table(table, source / f"{name}.parquet")
+ schema = api.DataSchema(
+ tuple(
+ api.TableSpec(
+ name,
+ "id",
+ {f.name: api.DType.from_arrow(f.type) for f in table.schema},
+ time_column="at" if name == "events" else api.NON_TEMPORAL,
+ )
+ for name, table in data.items()
+ ),
+ (
+ api.RelationSpec("orders", "users", "id", "events", "user"),
+ api.RelationSpec("reviews", "events", "reviewer", "users", "id"),
+ api.RelationSpec("parent", "events", "parent", "events", "id"),
+ ),
+ )
+ dataset = api.Dataset({name: api.Parquet(str(source / f"{name}.parquet")) for name in data})
+ index = tmp_path / "index"
+ spec = contract(schema)
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1)):
+ pass
+ anchor_path = tmp_path / "anchors.parquet"
+ pq.write_table(
+ pa.table(
+ {
+ "user": [-2, 7],
+ "at": pa.array([20, 20], pa.timestamp("s")),
+ "sample": ["left", "right"],
+ }
+ ),
+ anchor_path,
+ )
+ anchors = api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at", "sample")
+ return index, source, dataset, spec, anchors
+
+
+def sample(engine, spec, anchors):
+ with api.prepare(contract=spec, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events = graph.nodes["events"]
+ assert set(events.ids.to_pylist()) == {"a", "b", "d"}
+ assert events.features["at"].type == pa.timestamp("s", tz="UTC")
+ assert set(events.features["value"].to_pylist()) == {2.0, 4.0, None}
+ assert graph.edges["orders"].source_type == "users"
+ assert graph.edges["orders"].destination_type == "events"
+ for left, right in zip(
+ *[graph.edges["orders"].index[n].to_pylist() for n in ("source", "destination")],
+ strict=True,
+ ):
+ assert graph.nodes["users"].owners[left] == events.owners[right]
+ return graph
+
+
+def test_relocated_index_without_sources_or_base_writes(indexed, tmp_path, monkeypatch):
+ index, source, dataset, spec, anchors = indexed
+ relocated = tmp_path / "relocated"
+ shutil.move(str(index), relocated)
+ shutil.rmtree(source)
+ before = inventory(relocated)
+ original_ingest = diskgraph.ingest
+ extensions = []
+
+ def ingest(*args, **kwargs):
+ assert kwargs.get("parent_index_path") == str(relocated)
+ assert not str(args[1]).startswith(str(relocated))
+ extensions.append(args[1])
+ return original_ingest(*args, **kwargs)
+
+ monkeypatch.setattr(diskgraph, "ingest", ingest)
+ with read_only(relocated):
+ engine = api.DiskGraphEngine.from_index(
+ relocated, work_dir=tmp_path / "work", num_threads=1
+ )
+ assert engine.schema == spec.schema
+ sample(engine, spec, anchors)
+ assert extensions
+ spec_path = tmp_path / "contract.json"
+ spec_path.write_text(spec.to_json())
+ subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ """
+import sys
+from pathlib import Path
+from kumo.pytorch import api
+index, work, spec, anchors = sys.argv[1:]
+engine = api.DiskGraphEngine.from_index(index, work_dir=work, num_threads=1)
+contract = api.DataLoaderSpec.from_json(Path(spec).read_text())
+with api.prepare(contract=contract, engine=engine) as plan:
+ with plan.materialize(api.AnchorTable(api.Parquet(anchors), 'user', 'at', 'sample')) as task:
+ graph = next(iter(task.batches(output_format='columnar'))).inputs['graph']
+ assert set(graph.nodes['events'].ids.to_pylist()) == {'a', 'b', 'd'}
+""",
+ str(relocated),
+ str(tmp_path / "fresh-process"),
+ str(spec_path),
+ anchors.source.path,
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ assert inventory(relocated) == before
+ assert all(not Path(path).exists() for path in extensions)
+ assert not (relocated / "tasks").exists()
+
+
+def test_snapshot_binding_needs_no_pytorch_manifest(indexed, tmp_path):
+ index, source, _, spec, anchors = indexed
+ config = (index / "diskgraph_config.json").read_text()
+ index = tmp_path / "engine-index"
+ diskgraph.ingest(config, str(index), col_stats=False, success_marker_content="{}")
+ # The real engine integration test covers connector -> GraphSnapshot creation.
+ snapshot = SimpleNamespace(index_path=index, id="graph@version", graph_config={})
+ engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=tmp_path / "work", num_threads=1)
+ shutil.rmtree(source)
+ before = inventory(index)
+ sample(engine, spec, anchors) # Custom logical relation aliases resolve by endpoint AND key.
+ assert inventory(index) == before
+
+
+def test_external_fitting_statistics_tokenizer_and_lengths(indexed, tmp_path):
+ index, _, _, spec, _ = indexed
+ before = inventory(index)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9)
+ assert stats.columns[("events", "value", "numerical")]["count"] == 3
+ assert stats.columns[("events", "value", "numerical")]["mean"] == 3.0
+ encoder = TableColumnEncoder(
+ api.Text(vocabulary_size=300, min_frequency=1),
+ api.DType("string"),
+ (("events", "note"),),
+ )
+ tokenizer, name = source.column_text(encoder, 20 * 10**9)
+ assert source.column_text(encoder, 20 * 10**9) == (tokenizer, name)
+ file = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9)
+ _, values = read_token_lengths(
+ index, {"file": file, "constant": 0}, cache_root=source._cache_root
+ )
+ assert len(values) == 4
+ assert inventory(index) == before
+
+
+def test_existing_base_statistics_are_reused_and_reported(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ from kumo.pytorch.column_stats import read_column_stats
+
+ requests = [("events", "value", "numerical")]
+ stored = read_column_stats(index, requests, 20, "seconds")
+ before = inventory(index)
+
+ def unexpected_compute(*args, **kwargs):
+ pytest.fail("the requested statistics already exist in the base")
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", unexpected_compute)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ reused = plan._source.column_stats(requests, 20 * 10**9)
+ assert reused.columns == stored.columns
+ assert reused.files == tuple(str(index / "col_stats" / f) for f in stored.files)
+ assert inventory(index) == before
+
+
+def test_external_statistics_compute_only_missing_requests(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ from kumo.pytorch.column_stats import read_column_stats
+
+ known = ("events", "value", "numerical")
+ missing = ("events", "value", "categorical")
+ stored = read_column_stats(index, [known], 20, "seconds")
+ before = inventory(index)
+ calls = []
+ original = diskgraph.cutoff_col_stats
+
+ def compute(index_path, request_json, **options):
+ request = json.loads(request_json)
+ assert request["columns"] == {"events": {"value": ["categorical"]}}
+ assert not Path(options["output_dir"]).is_relative_to(index)
+ calls.append(request)
+ return original(index_path, request_json, **options)
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(
+ pool.map(lambda _: source.column_stats([known, missing], 20 * 10**9), range(2))
+ )
+ for result in results:
+ assert result.columns[known]["mean"] == stored.columns[known]["mean"]
+ assert sorted(result.columns[missing]["category_counts"]) == [["2", 1], ["4", 1]]
+ count = len(calls)
+ assert count >= 1
+ assert source.column_stats([known, missing], 20 * 10**9) == results[0]
+ assert len(calls) == count
+ assert inventory(index) == before
+
+
+@pytest.mark.parametrize(
+ "unit,clock", [("ms", "milliseconds"), ("us", "microseconds"), ("ns", "nanoseconds")]
+)
+def test_native_clock_precision_preserves_cutoff(indexed, tmp_path, unit, clock):
+ index, source, _, spec, _ = indexed
+ path = source / "events.parquet"
+ table = pq.read_table(path)
+ table = table.set_column(
+ table.schema.get_field_index("at"),
+ "at",
+ pa.array([1001, 1002, 1003, 1001], pa.timestamp(unit)),
+ )
+ pq.write_table(table, path)
+ config = json.loads((index / "diskgraph_config.json").read_text())
+ config["timestamp_unit"] = clock
+ native = tmp_path / "native"
+ diskgraph.ingest(json.dumps(config), str(native), col_stats=False, success_marker_content="{}")
+ shutil.rmtree(source)
+ anchor_path = tmp_path / "precise-anchors.parquet"
+ pq.write_table(
+ pa.table({"user": [-2, 7], "at": pa.array([1002, 1002], pa.timestamp(unit))}), anchor_path
+ )
+ engine = api.DiskGraphEngine.from_index(native, work_dir=tmp_path / "work", num_threads=1)
+ schema = replace(engine.schema, relations=spec.schema.relations)
+ with api.prepare(contract=contract(schema), engine=engine) as plan:
+ with plan.materialize(api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at")) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events = graph.nodes["events"]
+ assert set(events.ids.to_pylist()) == {"a", "b", "d"}
+ assert events.features["at"].type == pa.timestamp(unit, tz="UTC")
+ assert set(events.features["at"].cast(pa.int64()).to_pylist()) == {1001, 1002}
+
+
+def test_concurrent_consumers_keep_task_cleanup_separate(indexed, tmp_path):
+ index, _, _, spec, anchors = indexed
+ before = inventory(index)
+
+ def run(_):
+ return sample(
+ api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "shared", num_threads=1),
+ spec,
+ anchors,
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert len(list(pool.map(run, range(2)))) == 2
+ assert inventory(index) == before
+
+
+def test_projection_alias_validation_and_bad_bindings(indexed, tmp_path):
+ index, _, dataset, spec, anchors = indexed
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+ assert api.check(spec, engine=engine) == ()
+ with pytest.raises(api.ValidationError, match="does not accept Dataset"):
+ api.prepare(spec, dataset, engine)
+ unresolved = replace(
+ spec,
+ schema=replace(
+ spec.schema,
+ tables=tuple(
+ replace(table, columns={name: None for name in table.columns})
+ for table in spec.schema.tables
+ ),
+ ),
+ )
+ with api.prepare(contract=unresolved, engine=engine) as plan:
+ assert plan.contract.schema == spec.schema
+ wrong = replace(spec.schema.relations[0], right_key="reviewer")
+ projected = replace(spec.schema, relations=(wrong,))
+ with api.prepare(contract=contract(projected), engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ edge = graph.edges["orders"]
+ pairs = {
+ (graph.nodes["users"].ids[left].as_py(), graph.nodes["events"].ids[right].as_py())
+ for left, right in zip(
+ edge.index["source"].to_pylist(),
+ edge.index["destination"].to_pylist(),
+ strict=True,
+ )
+ }
+ assert pairs == {(-2, "d"), (7, "a"), (7, "b")}
+ missing = replace(wrong, right_key="value")
+ assert api.check(
+ replace(spec, schema=replace(spec.schema, relations=(missing,))), engine=engine
+ )
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=index / "work")
+
+
+def test_physically_present_self_relation(indexed, tmp_path):
+ index, source, _, spec, anchors = indexed
+ shutil.rmtree(source)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ spec = replace(
+ spec,
+ sample=replace(
+ spec.sample,
+ neighborhoods={
+ "history": api.NeighborhoodSpec(
+ (
+ api.GraphLayer((api.GraphSample("orders", fanout=1, strategy="recent"),)),
+ api.GraphLayer((api.GraphSample("parent", fanout=1, strategy="recent"),)),
+ )
+ ),
+ },
+ ),
+ )
+ with api.prepare(contract=spec, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events, edge = graph.nodes["events"], graph.edges["parent"]
+ assert {
+ (events.ids[left].as_py(), events.ids[right].as_py())
+ for left, right in zip(
+ edge.index["source"].to_pylist(),
+ edge.index["destination"].to_pylist(),
+ strict=True,
+ )
+ } == {("b", "a")}
+
+
+@pytest.mark.parametrize(
+ "change,match",
+ [
+ ("legacy", "diskgraph_config"),
+ ("incomplete", "completion manifest"),
+ ("edge", "edge table"),
+ ("list", "unsupported stored type"),
+ ("pair", "edge pair"),
+ ],
+)
+def test_incompatible_artifacts_fail_before_sampling(indexed, tmp_path, change, match):
+ index, _, _, _, _ = indexed
+ if change == "legacy":
+ (index / "diskgraph_config.json").unlink()
+ elif change == "incomplete":
+ (index / "index_manifest.json").unlink()
+ else:
+ filename = "index_metadata.json" if change == "pair" else "diskgraph_config.json"
+ path = index / filename
+ value = json.loads(path.read_text())
+ if change == "edge":
+ value["tables"][0]["edge_table"] = {"src_table": "users", "dst_table": "users"}
+ elif change == "list":
+ value["tables"][0]["columns"][0]["ingested_dtype"] = "string_array"
+ else:
+ value["edge_relations"][0]["reverse_of"] = "absent"
+ path.write_text(json.dumps(value))
+ # Keep the native size inventory internally consistent for this semantic refusal.
+ manifest_path = index / "_MANIFEST.json"
+ if manifest_path.exists():
+ manifest = json.loads(manifest_path.read_text())
+ for entry in manifest["files"]:
+ if entry["rel_path"] == filename:
+ entry["size_bytes"] = path.stat().st_size
+ manifest_path.write_text(json.dumps(manifest))
+ with pytest.raises((ValueError, RuntimeError), match=match):
+ api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+
+
+def test_connector_versions_allow_restage_but_reject_changes(indexed, tmp_path):
+ legacy, source, dataset, spec, _ = indexed
+ versions = {
+ name: {"source": f"connector/catalog/{name}", "version": "v1"} for name in dataset.tables
+ }
+ path = tmp_path / "versioned"
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=path, source_versions=versions)):
+ pass
+ # Version-aware reuse explicitly trusts the connector, even if its token is stale.
+ # A caller that needs file-change detection must keep the default legacy binding.
+ event_path = source / "events.parquet"
+ events = pq.read_table(event_path)
+ pq.write_table(
+ events.set_column(
+ events.schema.get_field_index("value"), "value", pa.array([200.0, 400.0, 10000.0, None])
+ ),
+ event_path,
+ )
+ with pytest.raises(ValueError, match="different dataset"):
+ api.prepare(spec, dataset, api.DiskGraphEngine(index=legacy))
+ moved = tmp_path / "new_sources"
+ shutil.move(source, moved)
+ restaged = api.Dataset(
+ {name: api.Parquet(str(moved / f"{name}.parquet")) for name in dataset.tables}
+ )
+ before = inventory(path)
+ with api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=versions)):
+ pass
+ assert inventory(path) == before
+ for field in ("source", "version"):
+ changed = {name: {**entry, field: "different"} for name, entry in versions.items()}
+ with pytest.raises(ValueError, match="different dataset"):
+ api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=changed))
+ with pytest.raises(ValueError, match="every dataset table"):
+ api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions={}))
+
+
+@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink"])
+@pytest.mark.parametrize("explicit_first", [False, True])
+def test_explicit_index_work_dir_reuses_legacy_state(
+ indexed, tmp_path, monkeypatch, alias, explicit_first
+):
+ index, _, dataset, spec, anchors = indexed
+ if alias == "relative":
+ monkeypatch.chdir(tmp_path)
+ work = Path("index")
+ elif alias == "symlink":
+ work = tmp_path / "index-alias"
+ work.symlink_to(index, target_is_directory=True)
+ else:
+ work = index
+ encoder = TableColumnEncoder(
+ api.Text(vocabulary_size=300, min_frequency=1),
+ api.DType("string"),
+ (("events", "note"),),
+ )
+
+ def run(explicit):
+ options = {"work_dir": work} if explicit else {}
+ engine = api.DiskGraphEngine(index=index, num_threads=1, **options)
+ with api.prepare(spec, dataset, engine) as plan:
+ source = plan._source
+ stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9)
+ tokenizer, name = source.column_text(encoder, 20 * 10**9)
+ lengths = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9)
+ _, values = read_token_lengths(index, {"file": lengths, "constant": 0})
+ assert all((index / "col_stats" / path).is_file() for path in stats.files)
+ assert (index / "col_stats" / name).is_file()
+ with plan.materialize(anchors) as task:
+ assert any((index / "train_tables").iterdir())
+ assert any((index / "tasks").iterdir())
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ assert set(graph.nodes["events"].ids.to_pylist()) == {"a", "b", "d"}
+ assert not list((index / "train_tables").iterdir())
+ assert not list((index / "tasks").iterdir())
+ return stats, tokenizer, name, lengths, values.tolist()
+
+ expected = run(explicit_first)
+ before = inventory(index)
+
+ def unexpected_compute(*args, **kwargs):
+ pytest.fail("existing in-index cache must be reused")
+
+ for function in ("cutoff_col_stats", "train_tokenizer", "token_lengths"):
+ monkeypatch.setattr(diskgraph, function, unexpected_compute)
+ assert run(not explicit_first) == expected
+ assert inventory(index) == before
+ assert not (index / "cache").exists()
+
+
+@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink", "nested"])
+def test_adopted_indexes_require_external_work_dir(indexed, tmp_path, monkeypatch, alias):
+ index, _, _, _, _ = indexed
+ if alias == "relative":
+ monkeypatch.chdir(tmp_path)
+ work = Path("index")
+ elif alias == "symlink":
+ work = tmp_path / "index-alias"
+ work.symlink_to(index, target_is_directory=True)
+ else:
+ work = index if alias == "absolute" else index / "work"
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=work)
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_snapshot(
+ SimpleNamespace(index_path=index, id="test"), work_dir=work
+ )
+ if alias == "nested":
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine(index=index, work_dir=work)
+
+
+@pytest.mark.parametrize("root", ["tasks", "train_tables", "cache"])
+@pytest.mark.parametrize("symlink", [False, True])
+def test_external_writable_roots_cannot_overlap_base(indexed, tmp_path, root, symlink):
+ index, _, _, _, _ = indexed
+ work = tmp_path / "work"
+ work.mkdir()
+ if symlink:
+ (work / root).symlink_to(index, target_is_directory=True)
+ else:
+ moved = work / root
+ shutil.move(index, moved)
+ index = moved
+ before = inventory(index)
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=work)
+ assert inventory(index) == before
+ assert set(p.name for p in work.iterdir()) == {root}
+
+
+def test_external_statistics_reuse_overlapping_requests(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ known = ("events", "value", "numerical")
+ missing = ("events", "value", "categorical")
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ initial = source.column_stats([known], 20 * 10**9)
+ original = diskgraph.cutoff_col_stats
+ calls = []
+
+ def compute(index_path, request_json, **options):
+ request = json.loads(request_json)
+ assert request["columns"] == {"events": {"value": ["categorical"]}}
+ calls.append(request)
+ return original(index_path, request_json, **options)
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute)
+ result = source.column_stats([known, missing], 20 * 10**9)
+ assert result.columns[known]["mean"] == initial.columns[known]["mean"]
+ assert result.files == initial.files
+ assert len(calls) == 1
+ assert source.column_stats([missing], 20 * 10**9).files == initial.files
+ assert len(calls) == 1
+
+
+def test_trusted_reuse_refuses_column_renames(indexed, tmp_path):
+ _, _, dataset, spec, _ = indexed
+ versions = {name: {"source": name, "version": "v1"} for name in dataset.tables}
+ index = tmp_path / "trusted"
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, source_versions=versions)):
+ pass
+ renamed = api.Dataset(
+ {
+ name: api.Parquet(source.path, columns={"value": "renamed"})
+ if name == "events"
+ else source
+ for name, source in dataset.tables.items()
+ }
+ )
+ with pytest.raises(ValueError, match="DG-COLUMN-RENAME"):
+ api.prepare(spec, renamed, api.DiskGraphEngine(index=index, source_versions=versions))
+
+
+def test_legacy_ingest_checks_the_snapshot_recorded_in_provenance(indexed, tmp_path, monkeypatch):
+ from kumo.pytorch import materialization
+
+ _, source, dataset, spec, _ = indexed
+ original = materialization._dataset_payload
+ calls = 0
+
+ def capture(*args, **kwargs):
+ nonlocal calls
+ payload = original(*args, **kwargs)
+ calls += 1
+ if calls == 1:
+ path = source / "events.parquet"
+ table = pq.read_table(path)
+ pq.write_table(table.slice(0, 2), path)
+ return payload
+
+ monkeypatch.setattr(materialization, "_dataset_payload", capture)
+ index = tmp_path / "new-index"
+ with pytest.raises(ValueError, match="changed while DiskGraph was ingesting"):
+ api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1))
+ assert not index.exists()
+
+
+def test_external_storage_refuses_old_native_api_before_task_writes(indexed, tmp_path, monkeypatch):
+ index, _, dataset, spec, _ = indexed
+
+ def old_cutoff_col_stats(index_dir, request_json, num_threads=None):
+ pytest.fail("compatibility check must not execute statistics")
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", old_cutoff_col_stats)
+ work = tmp_path / "work"
+ engine = api.DiskGraphEngine.from_index(index, work_dir=work)
+ with pytest.raises(RuntimeError, match="matching kumo-engine revision"):
+ api.prepare(contract=spec, engine=engine)
+ assert not work.exists()
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index)):
+ pass
+
+
+def test_index_local_statistics_read_each_file_once(indexed, tmp_path, monkeypatch):
+ from kumo.pytorch.column_stats import read_column_stats
+
+ index, _, _, _, _ = indexed
+ request = [("events", "value", "numerical")]
+ expected = read_column_stats(index, request, 20, "seconds")
+ alias = tmp_path / "alias"
+ alias.symlink_to(index, target_is_directory=True)
+ original = Path.read_text
+ reads = []
+
+ def read(path, *args, **kwargs):
+ if path.name.endswith(".col_stats.json"):
+ reads.append(path)
+ return original(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "read_text", read)
+ assert read_column_stats(index, request, 20, "seconds", cache_root=alias) == expected
+ assert len(reads) == 1
diff --git a/packages/kumo-engine/test/test_pytorch_snapshot.py b/packages/kumo-engine/test/test_pytorch_snapshot.py
new file mode 100644
index 000000000..6df130c83
--- /dev/null
+++ b/packages/kumo-engine/test/test_pytorch_snapshot.py
@@ -0,0 +1,168 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Optional cross-package qualification of the installed PyTorch loader."""
+
+import hashlib
+import shutil
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from kumoapi.graph import Edge
+from kumoapi.typing import Stype
+
+import kumo.engine as kumo
+from kumo.engine._scratch import get_scratch_dir
+
+api = pytest.importorskip('kumo.pytorch.api')
+
+
+def _files(root):
+ return {
+ str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in root.rglob('*') if p.is_file()
+ }
+
+
+def test_connector_snapshot_columnar_export_reload(tmp_path, monkeypatch):
+ data = tmp_path / 'sources'
+ data.mkdir()
+ pq.write_table(pa.table({
+ 'id': [-2, 7],
+ 'name': ['Alice', 'Bob']
+ }), data / 'users.parquet')
+ pq.write_table(
+ pa.table({
+ 'id': [1, 2, 3, 4],
+ 'user': [-2, -2, -2, 7],
+ 'at': pa.array([10, 20, 30, 10], pa.timestamp('s')),
+ 'value': [2.0, 4.0, 100.0, None],
+ }),
+ data / 'events.parquet',
+ )
+ connector = kumo.FileConnector(root_dir=str(data))
+ graph = kumo.Graph(
+ tables={
+ 'users':
+ kumo.Table.from_source_table(connector['users.parquet'],
+ primary_key='id'),
+ 'events':
+ kumo.Table.from_source_table(connector['events.parquet'],
+ primary_key='id',
+ time_column='at').with_column(
+ 'user', stype=Stype.ID),
+ },
+ edges=[Edge(src_table='events', fkey='user', dst_table='users')],
+ )
+ previous = get_scratch_dir()
+ try:
+ kumo.configure(scratch_dir=tmp_path / 'producer')
+ snapshot = graph.snapshot()
+ assert not (snapshot.index_path / 'index_manifest.json').exists()
+ anchor_path = tmp_path / 'anchors.parquet'
+ pq.write_table(
+ pa.table({
+ 'user': [-2, 7],
+ 'at': pa.array([20, 20], pa.timestamp('s'))
+ }),
+ anchor_path,
+ )
+ anchors = api.AnchorTable(api.Parquet(str(anchor_path)), 'user', 'at')
+
+ def run(snapshot, work):
+ before = _files(snapshot.index_path)
+ engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=work,
+ num_threads=1)
+ relation = engine.schema.relations[0]
+ contract = api.DataLoaderSpec(
+ engine.schema,
+ api.SampleSpec(
+ entity=api.EntitySpec('users'),
+ neighborhoods={
+ 'history':
+ api.NeighborhoodSpec((api.GraphLayer((api.GraphSample(
+ relation.name,
+ fanout=2,
+ direction='reverse',
+ strategy='recent',
+ ), )), )),
+ },
+ ),
+ api.BatchSpec(
+ inputs={
+ 'graph':
+ api.Input(
+ api.NeighborhoodSource('history'),
+ api.Graph(
+ node_fields={
+ table.name:
+ api.Fields(
+ tuple(
+ api.Encode(name, api.Raw())
+ for name in table.columns))
+ for table in engine.schema.tables
+ }),
+ None,
+ )
+ },
+ batching=api.Batching(max_samples=2),
+ ),
+ )
+ with (
+ api.prepare(contract=contract, engine=engine) as plan,
+ plan.materialize(anchors) as task,
+ ):
+ batch = next(iter(task.batches(output_format='columnar')))
+ out = batch.inputs['graph']
+ events = out.nodes['events']
+ assert set(events.ids.to_pylist()) == {1, 2, 4}
+ assert set(
+ events.features['value'].to_pylist()) == {2.0, 4.0, None}
+ assert relation.name in out.edges
+ edge = out.edges[relation.name]
+ pairs = {(out.nodes[edge.source_type].ids[left].as_py(),
+ out.nodes[edge.destination_type].ids[right].as_py())
+ for left, right in zip(
+ edge.index['source'].to_pylist(),
+ edge.index['destination'].to_pylist(), strict=True)
+ }
+ expected_pairs = {(-2, 1), (-2, 2), (7, 4)}
+ if edge.source_type == 'events':
+ expected_pairs = {(event, user)
+ for user, event in expected_pairs}
+ assert pairs == expected_pairs
+ for edge in out.edges.values():
+ for left, right in zip(
+ edge.index['source'].to_pylist(),
+ edge.index['destination'].to_pylist(),
+ strict=True,
+ ):
+ assert (out.nodes[edge.source_type].owners[left] ==
+ out.nodes[edge.destination_type].owners[right])
+ result = events.features.to_pylist()
+ assert _files(snapshot.index_path) == before
+ return result, engine._binding.identity
+
+ expected, identity = run(snapshot, tmp_path / 'work1')
+ exports = kumo.FileConnector(root_dir=str(tmp_path / 'exports'))
+ location = snapshot.export(kumo.SnapshotOutputConfig(exports, 'saved'))
+ shutil.rmtree(data)
+ shutil.rmtree(tmp_path / 'producer')
+ kumo.configure(scratch_dir=tmp_path / 'consumer')
+ loaded = kumo.GraphSnapshot.load_from_connector(exports, path=location)
+ import diskgraph
+
+ original = diskgraph.ingest
+
+ def extension_only(*args, **kwargs):
+ assert kwargs.get('parent_index_path') == str(loaded.index_path)
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(diskgraph, 'ingest', extension_only)
+ assert run(loaded, tmp_path / 'work2') == (expected, identity)
+ assert (api.DiskGraphEngine.from_index(
+ loaded.index_path,
+ work_dir=tmp_path / 'work3')._binding.identity == identity)
+ finally:
+ kumo.configure(scratch_dir=previous)