From 4304b2c5159c60651f474ec54df4c12288b3fb35 Mon Sep 17 00:00:00 2001
From: Alex Porter <102990994+am-porter@users.noreply.github.com>
Date: Wed, 18 Nov 2026 14:59:20 -0800
Subject: [PATCH 1/3] Bind kumo-pytorch to existing DiskGraph snapshots
---
.github/workflows/kumo-pytorch-diskgraph.yml | 33 ++
.../kumo-pytorch/docs/04-engines-and-data.md | 65 ++-
.../kumo-pytorch/docs/api/README.md | 2 +-
.../kumo-pytorch/docs/api/engines.md | 50 ++
.../kumo-pytorch/docs/api/lifecycle.md | 16 +-
.../src/kumo/pytorch/column_stats.py | 49 +-
.../kumo-pytorch/src/kumo/pytorch/engine.py | 7 +-
.../src/kumo/pytorch/engines/diskgraph.py | 134 ++++-
.../kumo/pytorch/engines/native_anchors.py | 8 +-
.../src/kumo/pytorch/index_binding.py | 279 ++++++++++
.../src/kumo/pytorch/materialization.py | 27 +-
.../src/kumo/pytorch/preparation.py | 39 +-
.../tests/conformance/test_fake_ingest.py | 15 +-
.../kumo-pytorch/tests/test_existing_index.py | 498 ++++++++++++++++++
.../kumo-engine/test/test_pytorch_snapshot.py | 155 ++++++
15 files changed, 1325 insertions(+), 52 deletions(-)
create mode 100644 packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py
create mode 100644 packages/experimental/kumo-pytorch/tests/test_existing_index.py
create mode 100644 packages/kumo-engine/test/test_pytorch_snapshot.py
diff --git a/.github/workflows/kumo-pytorch-diskgraph.yml b/.github/workflows/kumo-pytorch-diskgraph.yml
index e2d4bbcc2..d0a5d4df5 100644
--- a/.github/workflows/kumo-pytorch-diskgraph.yml
+++ b/.github/workflows/kumo-pytorch-diskgraph.yml
@@ -14,6 +14,11 @@ on: # yamllint disable-line rule:truthy
pull_request:
paths:
- 'packages/experimental/kumo-pytorch/**'
+ - 'packages/kumo-engine/**'
+ - 'packages/kumo-core/**'
+ - 'packages/kumo-api/**'
+ - 'packages/kumo-pql/**'
+ - 'packages/kumo-settings/**'
- 'packages/diskgraph/**'
- 'scripts/diskgraph'
- '.github/workflows/_diskgraph-wheel.yml'
@@ -23,6 +28,11 @@ on: # yamllint disable-line rule:truthy
branches: [main]
paths:
- 'packages/experimental/kumo-pytorch/**'
+ - 'packages/kumo-engine/**'
+ - 'packages/kumo-core/**'
+ - 'packages/kumo-api/**'
+ - 'packages/kumo-pql/**'
+ - 'packages/kumo-settings/**'
- 'packages/diskgraph/**'
- 'scripts/diskgraph'
- '.github/workflows/_diskgraph-wheel.yml'
@@ -150,3 +160,26 @@ jobs:
run: |
../fresh/bin/python -m pytest -p no:cacheprovider -m notebook --require-imports -q \
--durations=0 tests/test_notebooks.py
+ - name: Build the engine and its workspace dependencies as wheels
+ working-directory: .
+ run: |
+ set -euo pipefail
+ for package in kumo-api kumo-pql kumo-settings kumo-core kumo-engine; do
+ uv build --wheel --package "$package" --out-dir "$RUNNER_TEMP/engine-wheels"
+ done
+ cp packages/kumo-engine/test/test_pytorch_snapshot.py "$RUNNER_TEMP/stage/tests/"
+ - name: Install the engine wheels alongside PyTorch and DiskGraph
+ working-directory: ${{ runner.temp }}
+ run: |
+ uv pip install --python fresh/bin/python --torch-backend cpu engine-wheels/*.whl
+ uv pip check --python fresh/bin/python
+ - name: Connector snapshot to columnar batches, from installed wheels
+ working-directory: ${{ runner.temp }}/stage
+ run: |
+ ../fresh/bin/python -I -c '
+ import pathlib, sys
+ import diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings
+ for module in (diskgraph, kumo.engine, kumo.pytorch, kumo_core, kumoapi, kumopql, kumo.settings):
+ assert pathlib.Path(module.__file__).is_relative_to(sys.prefix), module.__file__
+ '
+ ../fresh/bin/python -m pytest -p no:cacheprovider -q --require-imports tests/test_pytorch_snapshot.py
diff --git a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
index 48752326f..d23cb4453 100644
--- a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
+++ b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
@@ -179,8 +179,9 @@ read.
The index holds rows, key maps and relations. It holds no anchors, labels or
fitted encoders, so several contracts over the same tables can share one index.
-Fitting keeps the column statistics of encoders over stored columns in
-`col_stats/`, so it needs the index directory to be writable. Each encoder
+By default, fitting keeps column statistics of encoders over stored columns in
+the index's `col_stats/`, so that directory must be writable. Supply an external
+`work_dir` to put newly computed statistics and task state there instead. Each encoder
cutoff has one directory there, with a `
.col_stats.json` file per table.
A later fit at the same cutoff computes only the columns and statistics those
files lack, and adds them. Ingest computes the statistics for the cutoffs in
@@ -195,6 +196,66 @@ it was built. With
`ingest="never"`, `prepare` requires an index that already exists, which is
what you usually want in a scoring job.
+### Open an existing snapshot or index
+
+`DiskGraphEngine.from_snapshot(snapshot, work_dir=...)` consumes an existing
+`kumo.engine.GraphSnapshot`. Use the engine's existing snapshot loader to
+restore an exported snapshot first. `DiskGraphEngine.from_index(index_path,
+work_dir=...)` opens the native index directory directly, including a
+PyTorch-produced index. Neither factory exports sources or ingests the base.
+
+
+```python
+engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir="/scratch/loader")
+contract = api.DataLoaderSpec(
+ schema=engine.schema,
+ sample=sample_spec,
+ batch=batch_spec,
+)
+with api.prepare(contract=contract, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ for batch in task.batches(output_format="columnar"):
+ graph = batch.inputs["graph"]
+```
+
+`SampleSpec` and `BatchSpec` retain their usual meaning. `engine.schema` derives
+stored types, keys, clocks and relationships from the existing native config
+and metadata. PyTorch-produced indexes retain their logical relation aliases;
+engine/native indexes initially use native FK names. A caller can provide a
+compatible schema projection and relation aliases: binding checks endpoints,
+FK columns and direction against the actual index. `prepare` rejects a Dataset
+alongside an existing-index engine. Anchors still select entity/time requests;
+neighborhoods are sampled during iteration.
+
+The base may be read-only. Task extensions, scalar-materialization scratch,
+new statistics, tokenizers and token-length caches go under the external work
+directory; existing immutable base caches can be read in place. Closing a task
+cleans up its temporary files. Derived caches remain for reuse. No new base
+metadata file is written. Existing snapshot identity or PyTorch producer
+fingerprints survive relocation. A bare completed native index without producer
+identity is hashed once at opening, which can be expensive for large artifacts.
+
+Opening requires the resolved `diskgraph_config.json` and the producer's
+completion metadata. Engine snapshots use the native inventory/integrity check;
+PyTorch indexes use their existing manifest, published after ingestion. Older
+artifacts missing resolved config are refused. Native edge-table layouts,
+end-time validity and unsupported stored types (including lists) are refused
+explicitly. Snapshot adoption does not recover relations omitted during ingestion
+or enable additional sampling strategies.
+
+For **source ingestion/reuse** after connector restaging, pass
+`DiskGraphEngine(index=..., work_dir=..., source_versions=...)`. Map every table
+to `{"source": "connector/source-coordinate", "version": "connector-version"}`.
+Equal trusted identities/versions and schema permit reuse across export paths;
+a changed identity/version or schema refuses reuse. Without trustworthy versions,
+keep the existing file-validation mode. This opt-in trusts the caller to change
+the version whenever source data changes; reuse does not check Parquet contents,
+file sizes or mtimes. Reusing a stale version token therefore reuses the old index.
+Physical file checks still detect source changes during initial ingestion.
+Opening an already materialized index
+never checks whether its sources now have a newer version; refreshing sources
+remains an explicit producer operation.
+
`output_format="model"` lets the DiskGraph engine build the batch's tensors
column by column, without building a Python record per event. It refuses raw types
that a model tensor can't hold (strings, dates, decimals) before fetching any
diff --git a/packages/experimental/kumo-pytorch/docs/api/README.md b/packages/experimental/kumo-pytorch/docs/api/README.md
index 9b54cb972..a9756b849 100644
--- a/packages/experimental/kumo-pytorch/docs/api/README.md
+++ b/packages/experimental/kumo-pytorch/docs/api/README.md
@@ -108,7 +108,7 @@ The [guide](../README.md) explains the concepts in order. The behavior spec, wit
| [`InputExposureWarning`](targets.md#inputexposurewarning) | `kumo.pytorch.api` | A masked value is also selected by an unmasked named input. |
| [`DataLoaderSpec`](lifecycle.md#dataloaderspec) | `kumo.pytorch.api` | Schema, sampled values, and explicitly encoded model outputs. |
| [`check`](lifecycle.md#check) | `kumo.pytorch.api` | Return everything wrong with this combination, without raising. |
-| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind the dataset through the engine. |
+| [`prepare`](lifecycle.md#prepare) | `kumo.pytorch.api` | Validate, then bind sources or an existing index through the engine. |
| [`preview`](lifecycle.md#preview) | `kumo.pytorch.api` | Build up to eight raw samples and one model-facing batch. |
| [`Plan`](lifecycle.md#plan) | `kumo.pytorch.api` | A contract bound to a dataset through an engine, before any anchors. |
| [`Task`](lifecycle.md#task) | `kumo.pytorch.api` | Anchors bound to a plan: what is iterated, served from, and written out. |
diff --git a/packages/experimental/kumo-pytorch/docs/api/engines.md b/packages/experimental/kumo-pytorch/docs/api/engines.md
index 60984b721..34d56354f 100644
--- a/packages/experimental/kumo-pytorch/docs/api/engines.md
+++ b/packages/experimental/kumo-pytorch/docs/api/engines.md
@@ -299,6 +299,8 @@ class DiskGraphEngine(
num_shards: int = 1,
ingest: Literal['if_missing', 'never'] = 'if_missing',
encoder_cutoffs: Sequence[str | int | datetime | date | None] = (),
+ work_dir: str | Path | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
)
```
@@ -330,6 +332,54 @@ step, before any data is read.
- `num_shards`: index shards.
- `ingest`: `"if_missing"` builds the index when absent; `"never"` requires an existing one. A mismatching manifest is an error, never permission to reuse.
- `encoder_cutoffs`: `encoder_cutoff` values whose column statistics an ingest computes.
+- `work_dir`: external task/cache directory; required by existing-index factories. Source-ingestion callers may omit it to retain the original index-local behavior.
+- `source_versions`: optional table-to-identity mapping for source ingestion/reuse. Each entry has nonempty `source` (connector plus source identity) and `version` strings. These trusted connector versions replace file path/size/mtime identity; missing versions are never treated as equal. Omit for legacy file validation. The caller must change the version whenever source data changes: reuse does not compare Parquet contents, file sizes or mtimes in this opt-in mode.
+
+#### `DiskGraphEngine.from_index`
+
+*classmethod*
+
+```python
+DiskGraphEngine.from_index(
+ index: str | Path,
+ *,
+ work_dir: str | Path,
+ num_threads: int | None = None,
+) -> DiskGraphEngine
+```
+
+Open a complete native index without its original sources or any base writes.
+
+Read schema/relations from existing metadata. Older indexes without the
+resolved native config are refused. Native edge tables and non-scalar
+stored columns are not supported. Bare indexes without producer identity
+are hashed once; engine provenance and PyTorch manifests avoid that scan.
+`work_dir` holds task extensions and derived caches outside the base.
+
+#### `DiskGraphEngine.from_snapshot`
+
+*classmethod*
+
+```python
+DiskGraphEngine.from_snapshot(
+ snapshot: Any,
+ *,
+ work_dir: str | Path,
+ num_threads: int | None = None,
+) -> DiskGraphEngine
+```
+
+Bind a kumo.engine GraphSnapshot using its existing index and identity.
+
+Uses public `index_path` and `id` fields; importing kumo.engine is
+unnecessary. Load exported snapshots through the existing engine loader.
+No source export, base ingestion or additional metadata file is needed.
+
+#### `DiskGraphEngine.schema`
+
+*property*
+
+Resolved stored schema for a from_index/from_snapshot engine.
#### `DiskGraphEngine.check_contract`
diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
index 85aefeb1d..264eabe86 100644
--- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
+++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md
@@ -144,8 +144,8 @@ A readable multi-line summary of the contract.
```python
def check(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
@@ -169,14 +169,18 @@ The whole contract must still be well formed.
```python
def prepare(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
inference: bool = False,
) -> Plan
```
-Validate, then bind the dataset through the engine.
+Validate, then bind sources or an existing index through the engine.
+
+Omit `dataset` only for a DiskGraphEngine created with `from_index` or
+`from_snapshot`. Its stored schema supplies omitted types; it never reads
+the original sources or rebuilds the base. `engine` is always required.
Every check that does not need data happens first, so an unexecutable
combination is refused before a row is read. Anchors, and everything fitted
@@ -221,7 +225,7 @@ Called as `preview(task, requests)` or
```python
class Plan(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
source: SampleSource,
*,
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
index d1cf37703..fac26686a 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
@@ -243,7 +243,7 @@ def token_lengths_file(
def read_token_lengths(
- index: str | Path, entry: Mapping[str, Any]
+ index: str | Path, entry: Mapping[str, Any], *, cache_root: str | Path | None = None
) -> tuple[int, np.ndarray | None]:
"""A table's per-event constant and its read-only per-node counts, indexed by node ID.
@@ -255,6 +255,8 @@ def read_token_lengths(
if entry["file"] is None:
return entry["constant"], None
path = Path(index, "col_stats", entry["file"])
+ if not path.is_file() and cache_root is not None:
+ path = Path(cache_root, "col_stats", entry["file"])
if not path.stat().st_size:
return entry["constant"], np.frombuffer(b"", dtype=" ColumnStats:
- """The requested statistics at a cutoff, first computing into the index those it lacks."""
+ """Read stored statistics, computing missing values into the selected cache root."""
import diskgraph
- directory = Path(index, "col_stats", _cutoff_dir(ticks, unit))
+ base = Path(index, "col_stats", _cutoff_dir(ticks, unit))
+ external = cache_root is not None and Path(cache_root).resolve() != Path(index).resolve()
+ destination = Path(cache_root) if external else Path(index)
+ # Isolate request sets; immutable base statistics need not be recomputed.
+ recipe = hashlib.sha256(json.dumps(sorted(set(requests))).encode()).hexdigest()[:16]
+ relative = _cutoff_dir(ticks, unit) + (f"/{recipe}" if external else "")
+ directory = destination / "col_stats" / relative
paths = {
table: directory / f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests
}
+ files = set()
+
def read() -> dict:
- return {
- table: json.loads(path.read_text())[table]
- for table, path in paths.items()
- if path.is_file()
- }
+ found = {}
+ for table, path in paths.items():
+ for candidate in (base / path.name, path):
+ if candidate.is_file():
+ files.add(candidate)
+ data = json.loads(candidate.read_text())[table]
+ for column, values in data.items():
+ found.setdefault(table, {}).setdefault(column, {}).update(values)
+ return found
found = read()
missing = [
- (table, column, stype)
- for table, column, stype in requests
- if _STAT_KEYS[stype] not in found.get(table, {}).get(column, {})
+ item
+ for item in requests
+ if _STAT_KEYS[item[2]] not in found.get(item[0], {}).get(item[1], {})
]
if missing:
request = stats_request(missing, [ticks], unit)
- diskgraph.cutoff_col_stats(str(index), json.dumps(request), num_threads=num_threads)
+ request["dirs"] = [relative]
+ options = {"output_dir": str(destination)} if external else {}
+ diskgraph.cutoff_col_stats(
+ str(index), json.dumps(request), num_threads=num_threads, **options
+ )
found = read()
return ColumnStats(
{item: found[item[0]][item[1]] for item in requests},
{"value": ticks, "unit": unit},
- tuple(f"{directory.name}/{path.name}" for _, path in sorted(paths.items())),
+ tuple(
+ str(path if external else path.relative_to(Path(index) / "col_stats"))
+ for path in sorted(files)
+ ),
)
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
index 87eab16dd..b81d83a4c 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engine.py
@@ -5,13 +5,14 @@
"""The engine interface: what an engine must provide, and how a caller picks one.
A :class:`~kumo.pytorch.dataset.Dataset` says where the data is;
-an :class:`Engine` says what reads it. Both are supplied by the caller, and
-neither names the other.
+an :class:`Engine` says what reads it. Source ingestion supplies both.
+An existing-index ``DiskGraphEngine`` supplies the data binding itself, so
+``prepare`` and ``check`` accept it without a ``Dataset``.
An engine is a factory, not a holder of data: :meth:`Engine.open` binds a
dataset and returns a :class:`SampleSource`. Construction-time configuration
(artifact directories, index paths) belongs to the engine; data binding belongs
-to the dataset.
+to the dataset or the explicitly adopted index.
Distributed execution is expressed by ``rank``/``world_size`` on
``Task.batches``.
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
index 1d7b7a10c..ce50935b2 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
@@ -73,6 +73,7 @@
from kumo.pytorch.engine import AnchorTask, GraphRoot, SampleSource, register_engine
from kumo.pytorch.engines.parquet_anchors import ParquetAnchors, check_parquet_anchors
from kumo.pytorch.engines.unsupported import DISKGRAPH_UNSUPPORTED, Unsupported
+from kumo.pytorch.index_binding import open_index
from kumo.pytorch.materialization import _DiskGraphIndex, ensure_diskgraph_index
from kumo.pytorch.spec import DataLoaderSpec, ValidationIssue
@@ -214,6 +215,14 @@ class DiskGraphEngine:
requires an existing one. A mismatching manifest is an error, never
permission to reuse.
:param encoder_cutoffs: ``encoder_cutoff`` values whose column statistics an ingest computes.
+ :param work_dir: external task/cache directory; required by existing-index factories.
+ Source-ingestion callers may omit it to retain the original index-local behavior.
+ :param source_versions: optional table-to-identity mapping for source ingestion/reuse.
+ Each entry has nonempty ``source`` (connector plus source identity) and ``version``
+ strings. These trusted connector versions replace file path/size/mtime identity;
+ missing versions are never treated as equal. Omit for legacy file validation.
+ The caller must change the version whenever source data changes: reuse does
+ not compare Parquet contents, file sizes or mtimes in this opt-in mode.
"""
name = "diskgraph"
@@ -228,12 +237,23 @@ def __init__(
num_shards: int = 1,
ingest: Literal["if_missing", "never"] = "if_missing",
encoder_cutoffs: Sequence[str | int | dt.datetime | dt.date | None] = (),
+ work_dir: str | Path | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
):
if ingest not in {"if_missing", "never"}:
raise ValueError("ingest must be 'if_missing' or 'never'")
if isinstance(encoder_cutoffs, str):
raise TypeError("encoder_cutoffs is a list of cutoffs; wrap a single one in a list")
self.index = Path(index)
+ self.work_dir = None if work_dir is None else Path(work_dir).resolve()
+ if self.work_dir is not None and self.work_dir.is_relative_to(self.index.resolve()):
+ raise ValueError("work_dir must be outside the base index")
+ self.source_versions = (
+ None
+ if source_versions is None
+ else {name: dict(entry) for name, entry in source_versions.items()}
+ )
+ self._binding = None
self.num_threads = num_threads
self.num_shards = num_shards
self.ingest = ingest
@@ -241,6 +261,66 @@ def __init__(
dict.fromkeys(cutoff_nanos(value) for value in encoder_cutoffs)
)
+ @classmethod
+ def from_index(
+ cls, index: str | Path, *, work_dir: str | Path, num_threads: int | None = None
+ ) -> DiskGraphEngine:
+ """Open a complete native index without its original sources or any base writes.
+
+ Read schema/relations from existing metadata. Older indexes without the
+ resolved native config are refused. Native edge tables and non-scalar
+ stored columns are not supported. Bare indexes without producer identity
+ are hashed once; engine provenance and PyTorch manifests avoid that scan.
+ ``work_dir`` holds task extensions and derived caches outside the base.
+ """
+ result = cls(index=index, ingest="never", work_dir=work_dir, num_threads=num_threads)
+ result._binding = open_index(index)
+ return result
+
+ @classmethod
+ def from_snapshot(
+ cls, snapshot: Any, *, work_dir: str | Path, num_threads: int | None = None
+ ) -> DiskGraphEngine:
+ """Bind a kumo.engine GraphSnapshot using its existing index and identity.
+
+ Uses public ``index_path`` and ``id`` fields; importing kumo.engine is
+ unnecessary. Load exported snapshots through the existing engine loader.
+ No source export, base ingestion or additional metadata file is needed.
+ """
+ result = cls(
+ index=snapshot.index_path, ingest="never", work_dir=work_dir, num_threads=num_threads
+ )
+ result._binding = open_index(snapshot.index_path, snapshot_id=snapshot.id)
+ return result
+
+ @property
+ def schema(self) -> DataSchema:
+ """Resolved stored schema for a from_index/from_snapshot engine."""
+ if self._binding is None:
+ raise ValueError("schema is available on from_index/from_snapshot engines")
+ return self._binding.schema
+
+ def _resolve_index_schema(self, schema: DataSchema) -> DataSchema:
+ if self._binding is None:
+ raise ValueError("dataset is required unless the engine binds an existing index")
+ return self._binding.resolve(schema)[0]
+
+ def _check_binding(self, dataset: Dataset | None, schema: DataSchema):
+ if self._binding is not None and dataset is not None:
+ return (
+ ValidationIssue(
+ "dataset",
+ "an existing-index engine does not accept Dataset",
+ "omit dataset; the index supplies the table binding",
+ ),
+ )
+ if dataset is None:
+ try:
+ self._resolve_index_schema(schema)
+ except ValueError as error:
+ return (ValidationIssue("schema", str(error)),)
+ return ()
+
def check_contract(self, contract: DataLoaderSpec) -> tuple[ValidationIssue, ...]:
"""Refuse the entries of `unsupported` the contract uses, before any data is read."""
issues = _sample_refusals(contract.sample)
@@ -394,11 +474,25 @@ def check_anchors(table: AnchorTable) -> tuple[ValidationIssue, ...]:
def open(
self,
- dataset: Dataset,
+ dataset: Dataset | None,
schema: DataSchema,
*,
column_stats: Sequence[StatRequest] = (),
) -> SampleSource:
+ if self._binding is not None:
+ if dataset is not None:
+ raise ValueError("an existing-index engine does not accept Dataset")
+ self._binding.validate_files()
+ schema, artifact = self._binding.resolve(schema)
+ return DiskGraphSource(
+ artifact,
+ schema,
+ num_threads=self.num_threads,
+ work_dir=self.work_dir,
+ identity=self._binding.identity,
+ )
+ if dataset is None:
+ raise ValueError("dataset is required for source ingestion")
ingests = bool(column_stats and self.encoder_cutoffs) and not self.index.exists()
artifact = ensure_diskgraph_index(
schema,
@@ -408,11 +502,13 @@ def open(
num_threads=self.num_threads,
if_missing=self.ingest == "if_missing",
column_stats=(column_stats, self.encoder_cutoffs) if ingests else None,
+ source_versions=self.source_versions,
)
return DiskGraphSource(
artifact,
schema,
num_threads=self.num_threads,
+ work_dir=self.work_dir,
)
def materialize_anchors(self, table: AnchorTable) -> AnchorTask:
@@ -430,6 +526,8 @@ def __init__(
schema: DataSchema,
*,
num_threads: int | None,
+ work_dir: Path | None = None,
+ identity: str | None = None,
) -> None:
try:
import diskgraph
@@ -445,6 +543,19 @@ def __init__(
self.schema = schema
self._index_path = artifact.path
+ self.artifact_identity = identity
+ self._work_dir = Path(artifact.path) if work_dir is None else work_dir
+ cache_id = (
+ identity
+ or hashlib.sha256(
+ json.dumps(dict(artifact.manifest), sort_keys=True).encode()
+ ).hexdigest()
+ )
+ self._cache_root = (
+ Path(artifact.path)
+ if work_dir is None
+ else work_dir / "cache" / hashlib.sha256(cache_id.encode()).hexdigest()
+ )
metadata = json.loads(Path(artifact.path, "index_metadata.json").read_text())
self._clock = metadata.get("timestamp_unit", "seconds")
self._num_threads = num_threads
@@ -1123,7 +1234,18 @@ def column_stats(self, requests: Sequence[StatRequest], cutoff: int | None) -> C
if cutoff is None
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
- return read_column_stats(self._index_path, requests, ticks, self._clock, self._num_threads)
+ return read_column_stats(
+ self._index_path,
+ requests,
+ ticks,
+ self._clock,
+ self._num_threads,
+ cache_root=self._cache_root,
+ )
+
+ def _cache_file(self, name: str) -> Path:
+ existing = Path(self._index_path, "col_stats", name)
+ return existing if existing.is_file() else self._cache_root / "col_stats" / name
def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[str, str]:
ticks = (
@@ -1132,7 +1254,7 @@ def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = encoder.tokenizer_file(ticks, self._clock)
- path = Path(self._index_path, "col_stats", name)
+ path = self._cache_file(name)
if path.is_file():
try:
document = json.loads(path.read_text())
@@ -1181,17 +1303,19 @@ def token_lengths(
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = token_lengths_file(table, columns, required, ticks, self._clock)
- path = Path(self._index_path, "col_stats", name)
+ path = self._cache_file(name)
if not path.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}")
self._diskgraph.token_lengths(
self._index_path,
table,
list(columns),
- str(path),
+ str(temporary),
required=list(required),
num_threads=self._num_threads,
)
+ temporary.replace(path)
return name
def fit_graph_fields(
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
index 32bb6ee96..cbb892d94 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py
@@ -74,8 +74,8 @@ def __init__(self, source: Any, sample: Any, table: AnchorTable, *, directory=No
self._source, self._sample, self._table = source, sample, table
self._feature_types = sample_value_types(source.schema, sample)
self._pa, self._pc = pa, pc
- parent = Path(source._index_path)
- (parent / "tasks").mkdir(exist_ok=True)
+ parent = Path(source._work_dir)
+ (parent / "tasks").mkdir(parents=True, exist_ok=True)
(parent / "train_tables").mkdir(exist_ok=True)
self._temporary = directory or tempfile.TemporaryDirectory(
prefix="task-", dir=parent / "tasks"
@@ -470,7 +470,9 @@ def bind_token_lengths(self, assembler, contract, statistics):
raise ValueError(
"event token costs do not match the designated input's fitted recipe"
)
- fixed, values = read_token_lengths(self._source._index_path, entry)
+ fixed, values = read_token_lengths(
+ self._source._index_path, entry, cache_root=self._source._cache_root
+ )
if values is not None:
self._token_maps.append(values)
self._event_costs[table] = dict(
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py
new file mode 100644
index 000000000..a830e0073
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/index_binding.py
@@ -0,0 +1,279 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Read existing DiskGraph artifacts without their original table sources."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from kumo.pytorch.contract.dtypes import DType
+from kumo.pytorch.contract.schema import NON_TEMPORAL, DataSchema, RelationSpec, TableSpec
+from kumo.pytorch.materialization import _DiskGraphIndex, _fingerprint, _foreign_key
+
+_CLOCKS = {"seconds": "s", "milliseconds": "ms", "microseconds": "us", "nanoseconds": "ns"}
+
+
+def _read(root: Path, name: str) -> dict:
+ try:
+ value = json.loads((root / name).read_text())
+ except (OSError, ValueError) as error:
+ raise ValueError(f"cannot open DiskGraph metadata {root / name}: {error}") from error
+ if not isinstance(value, dict):
+ raise ValueError(f"DiskGraph metadata {root / name} must be an object")
+ return value
+
+
+def _dtype(column: dict, clock: str, table: str) -> DType:
+ name = column["ingested_dtype"]
+ name = {"date": "date32", "utf8": "string"}.get(name, name.replace("u_int", "uint"))
+ try:
+ if name == "timestamp":
+ return DType(name, unit=_CLOCKS[clock], timezone="UTC")
+ if name.startswith("decimal"):
+ return DType(name, precision=column["precision"], scale=column["scale"])
+ if column.get("semantic_type") == "multicategorical":
+ raise ValueError("multicategorical storage requires list support")
+ return DType(name)
+ except (KeyError, ValueError) as error:
+ raise ValueError(
+ f"unsupported stored type for {table}.{column['name']}: {column}"
+ ) from error
+
+
+@dataclass(frozen=True)
+class IndexBinding:
+ artifact: _DiskGraphIndex
+ schema: DataSchema
+ identity: str
+ metadata_fingerprint: str
+ physical_relations: tuple[tuple[RelationSpec, dict], ...]
+
+ def resolve(self, schema: DataSchema) -> tuple[DataSchema, _DiskGraphIndex]:
+ """Validate a projection and map consumer relation aliases to physical edges."""
+ stored = {table.name: table for table in self.schema.tables}
+ tables = []
+ for table in schema.tables:
+ actual = stored.get(table.name)
+ if actual is None:
+ raise ValueError(f"index has no node table {table.name!r}")
+ if (table.primary_key, table.time_column) != (actual.primary_key, actual.time_column):
+ raise ValueError(f"index keys/clock differ for table {table.name!r}")
+ columns = actual.columns if table.columns is None else table.columns
+ resolved = {}
+ for name, dtype in columns.items():
+ if name not in actual.columns:
+ raise ValueError(f"index has no column {table.name}.{name}")
+ expected = actual.columns[name]
+ if dtype is not None and dtype != expected:
+ raise ValueError(
+ f"index stored type differs for {table.name}.{name}: {expected}"
+ )
+ resolved[name] = expected
+ tables.append(replace(table, columns=resolved))
+ mapping = {}
+ definitions = {table.name: table for table in tables}
+ for relation in schema.relations:
+ try:
+ child, fk, parent, forward = _foreign_key(relation, definitions)
+ except KeyError as error:
+ raise ValueError(
+ f"relation {relation.name!r} refers to an undeclared table"
+ ) from error
+ matches = [
+ native
+ for physical, native in self.physical_relations
+ if (physical.left_table, physical.left_key, physical.right_table)
+ == (child, fk, parent)
+ ]
+ if len(matches) != 1:
+ raise ValueError(f"index has no unambiguous stored relation for {relation.name!r}")
+ native = matches[0]
+ mapping[relation.name] = {
+ "left_table": relation.left_table,
+ "right_table": relation.right_table,
+ "forward": native["forward" if forward else "reverse"],
+ "reverse": native["reverse" if forward else "forward"],
+ }
+ return DataSchema(tuple(tables), schema.relations), _DiskGraphIndex(
+ self.artifact.path, {"relation_mapping": mapping}
+ )
+
+ def validate_files(self) -> None:
+ root = Path(self.artifact.path)
+ if _metadata_fingerprint(root) != self.metadata_fingerprint:
+ raise ValueError("DiskGraph index metadata changed after binding; open it again")
+ _check_complete(root)
+
+
+def _check_complete(root: Path) -> None:
+ import diskgraph
+
+ if (root / "_MANIFEST.json").is_file():
+ diskgraph.verify_snapshot_integrity(str(root))
+ elif (root / "index_manifest.json").is_file():
+ # PyTorch publishes this manifest by atomic rename after native ingest;
+ # that producer does not request engine _MANIFEST/_SUCCESS markers.
+ _DiskGraphIndex.open(root)
+ else:
+ raise ValueError(f"incomplete DiskGraph index: {root} has no completion manifest")
+
+
+def _metadata_fingerprint(root: Path) -> str:
+ return _fingerprint(
+ {
+ name: hashlib.sha256((root / name).read_bytes()).hexdigest()
+ for name in (
+ "diskgraph_config.json",
+ "index_metadata.json",
+ "_MANIFEST.json",
+ "index_manifest.json",
+ "_provenance.json",
+ )
+ if (root / name).is_file()
+ }
+ )
+
+
+def open_index(path: str | Path, *, snapshot_id: str | None = None) -> IndexBinding:
+ """Build an in-memory binding; write nothing and never follow stored source paths."""
+ root = Path(path).resolve()
+ if not (root / "diskgraph_config.json").is_file():
+ raise ValueError(
+ f"{root} has no diskgraph_config.json; automatic opening of this legacy "
+ "index is unsupported (no schema is inferred from unavailable sources)"
+ )
+ _check_complete(root)
+ config, metadata = _read(root, "diskgraph_config.json"), _read(root, "index_metadata.json")
+ clock = metadata.get("timestamp_unit", "seconds")
+ if clock not in _CLOCKS or config.get("timestamp_unit", "seconds") != clock:
+ raise ValueError(f"unsupported or inconsistent DiskGraph clock: {clock!r}")
+ nodes = {node["name"]: node for node in metadata["node_types"]}
+ tables = []
+ for table in config["tables"]:
+ name = table["name"]
+ if table.get("edge_table") is not None:
+ raise ValueError(f"native edge table {name!r} is not supported by the PyTorch binding")
+ node = nodes.get(name)
+ if node is None:
+ raise ValueError(f"config table {name!r} is absent from native node metadata")
+ key, time = table.get("primary_key"), table.get("create_time")
+ if key != (node.get("primary_key_map") or {}).get("column"):
+ raise ValueError(f"inconsistent native primary key for {name!r}")
+ if time != node.get("create_time_col") or table.get("end_time"):
+ raise ValueError(f"inconsistent or unsupported native clock/end_time for {name!r}")
+ tables.append(
+ TableSpec(
+ name,
+ key,
+ {col["name"]: _dtype(col, clock, name) for col in table["columns"]},
+ time_column=time or NON_TEMPORAL,
+ )
+ )
+ if set(nodes) != {table.name for table in tables}:
+ raise ValueError("native node types differ from the resolved config")
+ by_name = {table.name: table for table in tables}
+ edges = {edge["name"]: edge for edge in metadata["edge_relations"]}
+ physical = []
+ for table in config["tables"]:
+ for fk in table.get("fkeys", []):
+ child, column, parent = table["name"], fk["src_column"], fk["dst_table"]
+ if parent not in by_name or column not in by_name[child].columns:
+ raise ValueError(f"invalid native foreign key {child}.{column} -> {parent}")
+ pk = by_name[parent].primary_key
+ # Native FK naming is defined by ingest; verify the name AND endpoints/pair.
+ name = f"{child}_{column}_to_{parent}_{pk}"
+ edge = edges.get(name)
+ reverse = edges.get((edge or {}).get("reverse_of"))
+ if (
+ edge is None
+ or reverse is None
+ or (edge["src_type"], edge["dst_type"]) != (child, parent)
+ or (reverse["src_type"], reverse["dst_type"]) != (parent, child)
+ or reverse.get("reverse_of") != name
+ ):
+ raise ValueError(f"missing or inconsistent native FK edge pair for {name!r}")
+ for direction in (edge, reverse):
+ target = by_name[direction["dst_type"]]
+ expected = (
+ None
+ if target.time_column == NON_TEMPORAL
+ else (target.name, target.time_column)
+ )
+ timestamp = (direction.get("dest_sort_key") or {}).get("timestamp")
+ actual = None if timestamp is None else (timestamp["table"], timestamp["column"])
+ if actual != expected:
+ raise ValueError(f"unsupported stored edge clock for {direction['name']!r}")
+ physical.append(
+ (
+ RelationSpec(name, child, column, parent, pk),
+ {
+ "left_table": child,
+ "right_table": parent,
+ "forward": name,
+ "reverse": reverse["name"],
+ },
+ )
+ )
+ legacy = _DiskGraphIndex.open(root).manifest if (root / "index_manifest.json").is_file() else {}
+ relations = []
+ for physical_relation, mapping in physical:
+ aliases = [
+ (alias, entry)
+ for alias, entry in legacy.get("relation_mapping", {}).items()
+ if entry["forward"] in (mapping["forward"], mapping["reverse"])
+ ]
+ if not aliases:
+ relations.append(physical_relation)
+ for alias, entry in aliases:
+ if entry["forward"] == mapping["forward"]:
+ relations.append(replace(physical_relation, name=alias))
+ else:
+ relations.append(
+ RelationSpec(
+ alias,
+ physical_relation.right_table,
+ physical_relation.right_key,
+ physical_relation.left_table,
+ physical_relation.left_key,
+ )
+ )
+ schema = DataSchema(tuple(tables), tuple(relations))
+ provenance = _read(root, "_provenance.json") if (root / "_provenance.json").is_file() else {}
+ recorded_id = provenance.get("id")
+ if snapshot_id and recorded_id and snapshot_id != recorded_id:
+ raise ValueError("snapshot identity differs from persisted provenance")
+ identity = snapshot_id or recorded_id
+ if not identity and legacy:
+ identity = _fingerprint(
+ {
+ key: legacy[key]
+ for key in ("dataset_fingerprint", "schema_fingerprint", "config_fingerprint")
+ }
+ )
+ if not identity:
+ # Bare native indexes have no guaranteed producer identity. Hash the existing
+ # artifact once, rather than equating same-sized files or inventing metadata.
+ digest = hashlib.sha256()
+ for entry in sorted(_read(root, "_MANIFEST.json")["files"], key=lambda e: e["rel_path"]):
+ file = root / entry["rel_path"]
+ if not file.resolve().is_relative_to(root):
+ raise ValueError("native manifest contains a path outside the index")
+ digest.update(entry["rel_path"].encode())
+ with file.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ identity = digest.hexdigest()
+ binding = IndexBinding(
+ _DiskGraphIndex(str(root), {}),
+ schema,
+ str(identity),
+ _metadata_fingerprint(root),
+ tuple(physical),
+ )
+ _, artifact = binding.resolve(schema)
+ return replace(binding, artifact=artifact)
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
index fc698a929..1f6bef1f3 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
@@ -56,9 +56,9 @@ def open(cls, path: str | Path) -> _DiskGraphIndex:
raise ValueError(f"{root} is missing DiskGraph index_metadata.json")
return cls(str(root), manifest)
- def validate_binding(self, schema: DataSchema, dataset: Dataset) -> None:
+ def validate_binding(self, schema: DataSchema, dataset: Dataset, source_versions=None) -> None:
expected_schema = _fingerprint(_schema_payload(schema))
- expected_dataset = _fingerprint(_dataset_payload(dataset))
+ expected_dataset = _fingerprint(_dataset_payload(dataset, source_versions))
if self.manifest.get("schema_fingerprint") != expected_schema:
raise ValueError("DiskGraph index was materialized from a different schema")
if self.manifest.get("dataset_fingerprint") != expected_dataset:
@@ -75,6 +75,7 @@ def ensure_diskgraph_index(
dataset_name: str = "default",
if_missing: bool = True,
column_stats: tuple[Sequence[StatRequest], Sequence[int | None]] | None = None,
+ source_versions: Mapping[str, Mapping[str, str]] | None = None,
) -> _DiskGraphIndex:
"""Reuse or build the base index while opening a DiskGraph engine.
@@ -95,7 +96,7 @@ def ensure_diskgraph_index(
output = Path(destination).resolve()
if output.exists():
artifact = _DiskGraphIndex.open(output)
- artifact.validate_binding(schema, dataset)
+ artifact.validate_binding(schema, dataset, source_versions)
_warn_dropped_rows(output)
return artifact
if not if_missing:
@@ -103,7 +104,8 @@ def ensure_diskgraph_index(
output.parent.mkdir(parents=True, exist_ok=True)
diskgraph = _import_diskgraph()
- source_snapshot = _dataset_payload(dataset)
+ source_snapshot = _dataset_payload(dataset, source_versions)
+ physical_snapshot = _dataset_payload(dataset)
config, relation_mapping, source_types = _compile_diskgraph_config(
schema,
dataset,
@@ -148,7 +150,7 @@ def ensure_diskgraph_index(
**thread_options,
)
elapsed = time.perf_counter() - started
- if _dataset_payload(dataset) != source_snapshot:
+ if _dataset_payload(dataset) != physical_snapshot:
raise ValueError("Parquet files changed while DiskGraph was ingesting them")
manifest = {
@@ -426,7 +428,20 @@ def _schema_payload(schema: DataSchema) -> dict[str, Any]:
}
-def _dataset_payload(dataset: Dataset) -> dict[str, Any]:
+def _dataset_payload(dataset: Dataset, source_versions=None) -> dict[str, Any]:
+ if source_versions is not None:
+ if set(source_versions) != set(dataset.tables):
+ raise ValueError("source_versions must identify every dataset table exactly once")
+ result = {}
+ for name, entry in source_versions.items():
+ if set(entry) != {"source", "version"} or any(
+ not isinstance(value, str) or not value for value in entry.values()
+ ):
+ raise ValueError(
+ "source_versions entries require nonempty source and version strings"
+ )
+ result[name] = dict(entry)
+ return {"source_versions": result}
payload = {}
for name, source in sorted(dataset.tables.items()):
if not isinstance(source, Parquet):
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
index 0d40d51d0..448de5184 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/preparation.py
@@ -69,8 +69,8 @@
def check(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
@@ -91,12 +91,14 @@ def check(
def _check(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
*,
anchors: AnchorTable | None = None,
inference: bool = False,
) -> tuple[DataLoaderSpec, tuple[ValidationIssue, ...]]:
+ if engine is None:
+ raise TypeError("engine is required")
entity_issues = _entity_key_issues(contract)
if entity_issues:
return contract, entity_issues
@@ -105,6 +107,8 @@ def _check(
try:
# Keep reporting declaration issues for a bad binding when types are
# already known, but never read metadata through a missing binding.
+ if dataset is None and not issues:
+ contract = replace(contract, schema=engine._resolve_index_schema(contract.schema))
contract = contract.resolve(dataset=None if issues else dataset)
except (ValueError, KeyError, OSError) as error:
if not issues:
@@ -163,7 +167,7 @@ class Plan:
def __init__(
self,
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
source: SampleSource,
*,
@@ -174,7 +178,9 @@ def __init__(
self.engine = engine
self.inference = inference
self._source: SampleSource | None = source
- self._dataset_fingerprint = _fingerprint(dataset)
+ self._dataset_fingerprint = getattr(source, "artifact_identity", None) or _fingerprint(
+ dataset
+ )
def materialize(
self,
@@ -272,8 +278,8 @@ def materialize(
from kumo.pytorch.aggregate_materialization import materialize_values
- parent = Path(source._index_path) / "tasks"
- parent.mkdir(exist_ok=True)
+ parent = Path(source._work_dir) / "tasks"
+ parent.mkdir(parents=True, exist_ok=True)
directory = tempfile.TemporaryDirectory(prefix="task-", dir=parent)
try:
values = materialize_values(
@@ -574,12 +580,16 @@ def reused_state(
def prepare(
contract: DataLoaderSpec,
- dataset: Dataset,
- engine: Engine,
+ dataset: Dataset | None = None,
+ engine: Engine | None = None,
*,
inference: bool = False,
) -> Plan:
- """Validate, then bind the dataset through the engine.
+ """Validate, then bind sources or an existing index through the engine.
+
+ Omit ``dataset`` only for a DiskGraphEngine created with ``from_index`` or
+ ``from_snapshot``. Its stored schema supplies omitted types; it never reads
+ the original sources or rebuilds the base. ``engine`` is always required.
Every check that does not need data happens first, so an unexecutable
combination is refused before a row is read. Anchors, and everything fitted
@@ -660,10 +670,17 @@ def preview(
def _binding_issues(
contract: DataLoaderSpec,
- dataset: Dataset,
+ dataset: Dataset | None,
engine: Engine,
) -> list[ValidationIssue]:
issues: list[ValidationIssue] = []
+ _check_binding = getattr(engine, "_check_binding", None)
+ if _check_binding is not None:
+ issues.extend(_check_binding(dataset, contract.schema))
+ if dataset is None:
+ if _check_binding is None:
+ issues.append(ValidationIssue("dataset", "this engine requires a Dataset"))
+ return issues
declared = [table.name for table in contract.schema.tables]
missing = dataset.missing_for(declared)
if missing:
diff --git a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
index 50a645713..7eb3aa953 100644
--- a/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
+++ b/packages/experimental/kumo-pytorch/tests/conformance/test_fake_ingest.py
@@ -86,7 +86,14 @@ class Shape:
class _Source:
"""In place of DiskGraphSource: what the engine hands the sampler."""
- def __init__(self, artifact: _DiskGraphIndex, schema: DataSchema, *, num_threads: int | None):
+ def __init__(
+ self,
+ artifact: _DiskGraphIndex,
+ schema: DataSchema,
+ *,
+ num_threads: int | None,
+ work_dir: Path | None = None,
+ ):
self.artifact, self.schema, self.num_threads = artifact, schema, num_threads
@@ -381,7 +388,11 @@ def _resize(path: Path) -> None:
@pytest.mark.parametrize("change", ("mtime", "size", "extra_file"))
def test_a_changed_dataset_is_refused_not_reused(change: str, tmp_path: Path) -> None:
- """M06: the dataset fingerprint covers each file's path, size and mtime."""
+ """M06: source ingestion without connector versions validates path/size/mtime.
+
+ Explicit existing-index adoption and version-aware restaging are covered
+ by test_existing_index; those modes do not require the original files.
+ """
with _built(tmp_path, parts=2) as (dataset, index):
before = _snapshot(index)
diff --git a/packages/experimental/kumo-pytorch/tests/test_existing_index.py b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
new file mode 100644
index 000000000..db17d67b0
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
@@ -0,0 +1,498 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Source-free, read-only reuse through the public preparation lifecycle."""
+
+import hashlib
+import json
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+
+from kumo.pytorch import api
+from kumo.pytorch.column_stats import TableColumnEncoder, read_token_lengths
+
+diskgraph = pytest.importorskip("diskgraph")
+
+
+def inventory(root):
+ return {
+ str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in root.rglob("*")
+ if p.is_file()
+ }
+
+
+@contextmanager
+def read_only(root):
+ paths = [root, *root.rglob("*")]
+ modes = {p: p.stat().st_mode for p in paths}
+ for p in paths:
+ p.chmod(0o555 if p.is_dir() else 0o444)
+ try:
+ yield
+ finally:
+ for p, mode in modes.items():
+ p.chmod(mode)
+
+
+def contract(schema):
+ return api.DataLoaderSpec(
+ schema,
+ api.SampleSpec(
+ entity=api.EntitySpec("users"),
+ neighborhoods={
+ "history": api.NeighborhoodSpec(
+ (api.GraphLayer((api.GraphSample("orders", fanout=2, strategy="recent"),)),)
+ ),
+ },
+ ),
+ api.BatchSpec(
+ inputs={
+ "graph": api.Input(
+ api.NeighborhoodSource("history"),
+ api.Graph(
+ node_fields={
+ table.name: api.Fields(
+ tuple(api.Encode(name, api.Raw()) for name in table.columns)
+ )
+ for table in schema.tables
+ }
+ ),
+ None,
+ )
+ },
+ batching=api.Batching(max_samples=2),
+ ),
+ )
+
+
+@pytest.fixture
+def indexed(tmp_path):
+ source = tmp_path / "sources"
+ source.mkdir()
+ data = {
+ "users": pa.table({"id": [-2, 7], "name": ["Alice", "Bob"]}),
+ "events": pa.table(
+ {
+ "id": ["a", "b", "c", "d"],
+ "user": [-2, -2, -2, 7],
+ "reviewer": [7, 7, 7, -2],
+ "parent": pa.array([None, "a", "b", None], pa.string()),
+ "at": pa.array([10, 20, 30, 10], pa.timestamp("s")),
+ "value": [2.0, 4.0, 100.0, None],
+ "note": ["first item", "second item", "future item", None],
+ }
+ ),
+ }
+ for name, table in data.items():
+ pq.write_table(table, source / f"{name}.parquet")
+ schema = api.DataSchema(
+ tuple(
+ api.TableSpec(
+ name,
+ "id",
+ {f.name: api.DType.from_arrow(f.type) for f in table.schema},
+ time_column="at" if name == "events" else api.NON_TEMPORAL,
+ )
+ for name, table in data.items()
+ ),
+ (
+ api.RelationSpec("orders", "users", "id", "events", "user"),
+ api.RelationSpec("reviews", "events", "reviewer", "users", "id"),
+ api.RelationSpec("parent", "events", "parent", "events", "id"),
+ ),
+ )
+ dataset = api.Dataset({name: api.Parquet(str(source / f"{name}.parquet")) for name in data})
+ index = tmp_path / "index"
+ spec = contract(schema)
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1)):
+ pass
+ anchor_path = tmp_path / "anchors.parquet"
+ pq.write_table(
+ pa.table(
+ {
+ "user": [-2, 7],
+ "at": pa.array([20, 20], pa.timestamp("s")),
+ "sample": ["left", "right"],
+ }
+ ),
+ anchor_path,
+ )
+ anchors = api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at", "sample")
+ return index, source, dataset, spec, anchors
+
+
+def sample(engine, spec, anchors):
+ with api.prepare(contract=spec, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events = graph.nodes["events"]
+ assert set(events.ids.to_pylist()) == {"a", "b", "d"}
+ assert events.features["at"].type == pa.timestamp("s", tz="UTC")
+ assert set(events.features["value"].to_pylist()) == {2.0, 4.0, None}
+ assert graph.edges["orders"].source_type == "users"
+ assert graph.edges["orders"].destination_type == "events"
+ for left, right in zip(
+ *[graph.edges["orders"].index[n].to_pylist() for n in ("source", "destination")],
+ strict=True,
+ ):
+ assert graph.nodes["users"].owners[left] == events.owners[right]
+ return graph
+
+
+def test_relocated_index_without_sources_or_base_writes(indexed, tmp_path, monkeypatch):
+ index, source, dataset, spec, anchors = indexed
+ relocated = tmp_path / "relocated"
+ shutil.move(str(index), relocated)
+ shutil.rmtree(source)
+ before = inventory(relocated)
+ original_ingest = diskgraph.ingest
+ extensions = []
+
+ def ingest(*args, **kwargs):
+ assert kwargs.get("parent_index_path") == str(relocated)
+ assert not str(args[1]).startswith(str(relocated))
+ extensions.append(args[1])
+ return original_ingest(*args, **kwargs)
+
+ monkeypatch.setattr(diskgraph, "ingest", ingest)
+ with read_only(relocated):
+ engine = api.DiskGraphEngine.from_index(
+ relocated, work_dir=tmp_path / "work", num_threads=1
+ )
+ assert engine.schema == spec.schema
+ sample(engine, spec, anchors)
+ assert extensions
+ spec_path = tmp_path / "contract.json"
+ spec_path.write_text(spec.to_json())
+ subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ """
+import sys
+from pathlib import Path
+from kumo.pytorch import api
+index, work, spec, anchors = sys.argv[1:]
+engine = api.DiskGraphEngine.from_index(index, work_dir=work, num_threads=1)
+contract = api.DataLoaderSpec.from_json(Path(spec).read_text())
+with api.prepare(contract=contract, engine=engine) as plan:
+ with plan.materialize(api.AnchorTable(api.Parquet(anchors), 'user', 'at', 'sample')) as task:
+ graph = next(iter(task.batches(output_format='columnar'))).inputs['graph']
+ assert set(graph.nodes['events'].ids.to_pylist()) == {'a', 'b', 'd'}
+""",
+ str(relocated),
+ str(tmp_path / "fresh-process"),
+ str(spec_path),
+ anchors.source.path,
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ assert inventory(relocated) == before
+ assert all(not Path(path).exists() for path in extensions)
+ assert not (relocated / "tasks").exists()
+
+
+def test_snapshot_binding_needs_no_pytorch_manifest(indexed, tmp_path):
+ index, source, _, spec, anchors = indexed
+ config = (index / "diskgraph_config.json").read_text()
+ index = tmp_path / "engine-index"
+ diskgraph.ingest(config, str(index), col_stats=False, success_marker_content="{}")
+ # The real engine integration test covers connector -> GraphSnapshot creation.
+ snapshot = SimpleNamespace(index_path=index, id="graph@version", graph_config={})
+ engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=tmp_path / "work", num_threads=1)
+ shutil.rmtree(source)
+ before = inventory(index)
+ sample(engine, spec, anchors) # Custom logical relation aliases resolve by endpoint AND key.
+ assert inventory(index) == before
+
+
+def test_external_fitting_statistics_tokenizer_and_lengths(indexed, tmp_path):
+ index, _, _, spec, _ = indexed
+ before = inventory(index)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9)
+ assert stats.columns[("events", "value", "numerical")]["count"] == 3
+ assert stats.columns[("events", "value", "numerical")]["mean"] == 3.0
+ encoder = TableColumnEncoder(
+ api.Text(vocabulary_size=300, min_frequency=1),
+ api.DType("string"),
+ (("events", "note"),),
+ )
+ tokenizer, name = source.column_text(encoder, 20 * 10**9)
+ assert source.column_text(encoder, 20 * 10**9) == (tokenizer, name)
+ file = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9)
+ _, values = read_token_lengths(
+ index, {"file": file, "constant": 0}, cache_root=source._cache_root
+ )
+ assert len(values) == 4
+ assert inventory(index) == before
+
+
+def test_existing_base_statistics_are_reused_and_reported(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ from kumo.pytorch.column_stats import read_column_stats
+
+ requests = [("events", "value", "numerical")]
+ stored = read_column_stats(index, requests, 20, "seconds")
+ before = inventory(index)
+
+ def unexpected_compute(*args, **kwargs):
+ pytest.fail("the requested statistics already exist in the base")
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", unexpected_compute)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ reused = plan._source.column_stats(requests, 20 * 10**9)
+ assert reused.columns == stored.columns
+ assert reused.files == tuple(str(index / "col_stats" / f) for f in stored.files)
+ assert inventory(index) == before
+
+
+def test_external_statistics_compute_only_missing_requests(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ from kumo.pytorch.column_stats import read_column_stats
+
+ known = ("events", "value", "numerical")
+ missing = ("events", "value", "categorical")
+ stored = read_column_stats(index, [known], 20, "seconds")
+ before = inventory(index)
+ calls = []
+ original = diskgraph.cutoff_col_stats
+
+ def compute(index_path, request_json, **options):
+ request = json.loads(request_json)
+ assert request["columns"] == {"events": {"value": ["categorical"]}}
+ assert not Path(options["output_dir"]).is_relative_to(index)
+ calls.append(request)
+ return original(index_path, request_json, **options)
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with read_only(index), api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(
+ pool.map(lambda _: source.column_stats([known, missing], 20 * 10**9), range(2))
+ )
+ for result in results:
+ assert result.columns[known]["mean"] == stored.columns[known]["mean"]
+ assert sorted(result.columns[missing]["category_counts"]) == [["2", 1], ["4", 1]]
+ count = len(calls)
+ assert count >= 1
+ assert source.column_stats([known, missing], 20 * 10**9) == results[0]
+ assert len(calls) == count
+ assert inventory(index) == before
+
+
+@pytest.mark.parametrize(
+ "unit,clock", [("ms", "milliseconds"), ("us", "microseconds"), ("ns", "nanoseconds")]
+)
+def test_native_clock_precision_preserves_cutoff(indexed, tmp_path, unit, clock):
+ index, source, _, spec, _ = indexed
+ path = source / "events.parquet"
+ table = pq.read_table(path)
+ table = table.set_column(
+ table.schema.get_field_index("at"),
+ "at",
+ pa.array([1001, 1002, 1003, 1001], pa.timestamp(unit)),
+ )
+ pq.write_table(table, path)
+ config = json.loads((index / "diskgraph_config.json").read_text())
+ config["timestamp_unit"] = clock
+ native = tmp_path / "native"
+ diskgraph.ingest(json.dumps(config), str(native), col_stats=False, success_marker_content="{}")
+ shutil.rmtree(source)
+ anchor_path = tmp_path / "precise-anchors.parquet"
+ pq.write_table(
+ pa.table({"user": [-2, 7], "at": pa.array([1002, 1002], pa.timestamp(unit))}), anchor_path
+ )
+ engine = api.DiskGraphEngine.from_index(native, work_dir=tmp_path / "work", num_threads=1)
+ schema = replace(engine.schema, relations=spec.schema.relations)
+ with api.prepare(contract=contract(schema), engine=engine) as plan:
+ with plan.materialize(api.AnchorTable(api.Parquet(str(anchor_path)), "user", "at")) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events = graph.nodes["events"]
+ assert set(events.ids.to_pylist()) == {"a", "b", "d"}
+ assert events.features["at"].type == pa.timestamp(unit, tz="UTC")
+ assert set(events.features["at"].cast(pa.int64()).to_pylist()) == {1001, 1002}
+
+
+def test_concurrent_consumers_keep_task_cleanup_separate(indexed, tmp_path):
+ index, _, _, spec, anchors = indexed
+ before = inventory(index)
+
+ def run(_):
+ return sample(
+ api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "shared", num_threads=1),
+ spec,
+ anchors,
+ )
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ assert len(list(pool.map(run, range(2)))) == 2
+ assert inventory(index) == before
+
+
+def test_projection_alias_validation_and_bad_bindings(indexed, tmp_path):
+ index, _, dataset, spec, anchors = indexed
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+ assert api.check(spec, engine=engine) == ()
+ with pytest.raises(api.ValidationError, match="does not accept Dataset"):
+ api.prepare(spec, dataset, engine)
+ unresolved = replace(
+ spec,
+ schema=replace(
+ spec.schema,
+ tables=tuple(
+ replace(table, columns={name: None for name in table.columns})
+ for table in spec.schema.tables
+ ),
+ ),
+ )
+ with api.prepare(contract=unresolved, engine=engine) as plan:
+ assert plan.contract.schema == spec.schema
+ wrong = replace(spec.schema.relations[0], right_key="reviewer")
+ projected = replace(spec.schema, relations=(wrong,))
+ with api.prepare(contract=contract(projected), engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ edge = graph.edges["orders"]
+ pairs = {
+ (graph.nodes["users"].ids[left].as_py(), graph.nodes["events"].ids[right].as_py())
+ for left, right in zip(
+ edge.index["source"].to_pylist(),
+ edge.index["destination"].to_pylist(),
+ strict=True,
+ )
+ }
+ assert pairs == {(-2, "d"), (7, "a"), (7, "b")}
+ missing = replace(wrong, right_key="value")
+ assert api.check(
+ replace(spec, schema=replace(spec.schema, relations=(missing,))), engine=engine
+ )
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=index / "work")
+
+
+def test_physically_present_self_relation(indexed, tmp_path):
+ index, source, _, spec, anchors = indexed
+ shutil.rmtree(source)
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ spec = replace(
+ spec,
+ sample=replace(
+ spec.sample,
+ neighborhoods={
+ "history": api.NeighborhoodSpec(
+ (
+ api.GraphLayer((api.GraphSample("orders", fanout=1, strategy="recent"),)),
+ api.GraphLayer((api.GraphSample("parent", fanout=1, strategy="recent"),)),
+ )
+ ),
+ },
+ ),
+ )
+ with api.prepare(contract=spec, engine=engine) as plan:
+ with plan.materialize(anchors) as task:
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ events, edge = graph.nodes["events"], graph.edges["parent"]
+ assert {
+ (events.ids[left].as_py(), events.ids[right].as_py())
+ for left, right in zip(
+ edge.index["source"].to_pylist(),
+ edge.index["destination"].to_pylist(),
+ strict=True,
+ )
+ } == {("b", "a")}
+
+
+@pytest.mark.parametrize(
+ "change,match",
+ [
+ ("legacy", "diskgraph_config"),
+ ("incomplete", "completion manifest"),
+ ("edge", "edge table"),
+ ("list", "unsupported stored type"),
+ ("pair", "edge pair"),
+ ],
+)
+def test_incompatible_artifacts_fail_before_sampling(indexed, tmp_path, change, match):
+ index, _, _, _, _ = indexed
+ if change == "legacy":
+ (index / "diskgraph_config.json").unlink()
+ elif change == "incomplete":
+ (index / "index_manifest.json").unlink()
+ else:
+ filename = "index_metadata.json" if change == "pair" else "diskgraph_config.json"
+ path = index / filename
+ value = json.loads(path.read_text())
+ if change == "edge":
+ value["tables"][0]["edge_table"] = {"src_table": "users", "dst_table": "users"}
+ elif change == "list":
+ value["tables"][0]["columns"][0]["ingested_dtype"] = "string_array"
+ else:
+ value["edge_relations"][0]["reverse_of"] = "absent"
+ path.write_text(json.dumps(value))
+ # Keep the native size inventory internally consistent for this semantic refusal.
+ manifest_path = index / "_MANIFEST.json"
+ if manifest_path.exists():
+ manifest = json.loads(manifest_path.read_text())
+ for entry in manifest["files"]:
+ if entry["rel_path"] == filename:
+ entry["size_bytes"] = path.stat().st_size
+ manifest_path.write_text(json.dumps(manifest))
+ with pytest.raises((ValueError, RuntimeError), match=match):
+ api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work")
+
+
+def test_connector_versions_allow_restage_but_reject_changes(indexed, tmp_path):
+ legacy, source, dataset, spec, _ = indexed
+ versions = {
+ name: {"source": f"connector/catalog/{name}", "version": "v1"} for name in dataset.tables
+ }
+ path = tmp_path / "versioned"
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=path, source_versions=versions)):
+ pass
+ # Version-aware reuse explicitly trusts the connector, even if its token is stale.
+ # A caller that needs file-change detection must keep the default legacy binding.
+ event_path = source / "events.parquet"
+ events = pq.read_table(event_path)
+ pq.write_table(
+ events.set_column(
+ events.schema.get_field_index("value"), "value", pa.array([200.0, 400.0, 10000.0, None])
+ ),
+ event_path,
+ )
+ with pytest.raises(ValueError, match="different dataset"):
+ api.prepare(spec, dataset, api.DiskGraphEngine(index=legacy))
+ moved = tmp_path / "new_sources"
+ shutil.move(source, moved)
+ restaged = api.Dataset(
+ {name: api.Parquet(str(moved / f"{name}.parquet")) for name in dataset.tables}
+ )
+ before = inventory(path)
+ with api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=versions)):
+ pass
+ assert inventory(path) == before
+ for field in ("source", "version"):
+ changed = {name: {**entry, field: "different"} for name, entry in versions.items()}
+ with pytest.raises(ValueError, match="different dataset"):
+ api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=changed))
+ with pytest.raises(ValueError, match="every dataset table"):
+ api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions={}))
diff --git a/packages/kumo-engine/test/test_pytorch_snapshot.py b/packages/kumo-engine/test/test_pytorch_snapshot.py
new file mode 100644
index 000000000..0b2de312a
--- /dev/null
+++ b/packages/kumo-engine/test/test_pytorch_snapshot.py
@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Optional cross-package qualification of the installed PyTorch loader."""
+
+import hashlib
+import shutil
+
+import pyarrow as pa
+import pyarrow.parquet as pq
+import pytest
+from kumoapi.graph import Edge
+from kumoapi.typing import Stype
+
+import kumo.engine as kumo
+from kumo.engine._scratch import get_scratch_dir
+
+api = pytest.importorskip('kumo.pytorch.api')
+
+
+def _files(root):
+ return {
+ str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in root.rglob('*') if p.is_file()
+ }
+
+
+def test_connector_snapshot_columnar_export_reload(tmp_path, monkeypatch):
+ data = tmp_path / 'sources'
+ data.mkdir()
+ pq.write_table(pa.table({
+ 'id': [-2, 7],
+ 'name': ['Alice', 'Bob']
+ }), data / 'users.parquet')
+ pq.write_table(
+ pa.table({
+ 'id': [1, 2, 3, 4],
+ 'user': [-2, -2, -2, 7],
+ 'at': pa.array([10, 20, 30, 10], pa.timestamp('s')),
+ 'value': [2.0, 4.0, 100.0, None],
+ }),
+ data / 'events.parquet',
+ )
+ connector = kumo.FileConnector(root_dir=str(data))
+ graph = kumo.Graph(
+ tables={
+ 'users':
+ kumo.Table.from_source_table(connector['users.parquet'],
+ primary_key='id'),
+ 'events':
+ kumo.Table.from_source_table(connector['events.parquet'],
+ primary_key='id',
+ time_column='at').with_column(
+ 'user', stype=Stype.ID),
+ },
+ edges=[Edge(src_table='events', fkey='user', dst_table='users')],
+ )
+ previous = get_scratch_dir()
+ try:
+ kumo.configure(scratch_dir=tmp_path / 'producer')
+ snapshot = graph.snapshot()
+ assert not (snapshot.index_path / 'index_manifest.json').exists()
+ anchor_path = tmp_path / 'anchors.parquet'
+ pq.write_table(
+ pa.table({
+ 'user': [-2, 7],
+ 'at': pa.array([20, 20], pa.timestamp('s'))
+ }),
+ anchor_path,
+ )
+ anchors = api.AnchorTable(api.Parquet(str(anchor_path)), 'user', 'at')
+
+ def run(snapshot, work):
+ before = _files(snapshot.index_path)
+ engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=work,
+ num_threads=1)
+ relation = engine.schema.relations[0]
+ contract = api.DataLoaderSpec(
+ engine.schema,
+ api.SampleSpec(
+ entity=api.EntitySpec('users'),
+ neighborhoods={
+ 'history':
+ api.NeighborhoodSpec((api.GraphLayer((api.GraphSample(
+ relation.name,
+ fanout=2,
+ direction='reverse',
+ strategy='recent',
+ ), )), )),
+ },
+ ),
+ api.BatchSpec(
+ inputs={
+ 'graph':
+ api.Input(
+ api.NeighborhoodSource('history'),
+ api.Graph(
+ node_fields={
+ table.name:
+ api.Fields(
+ tuple(
+ api.Encode(name, api.Raw())
+ for name in table.columns))
+ for table in engine.schema.tables
+ }),
+ None,
+ )
+ },
+ batching=api.Batching(max_samples=2),
+ ),
+ )
+ with (
+ api.prepare(contract=contract, engine=engine) as plan,
+ plan.materialize(anchors) as task,
+ ):
+ batch = next(iter(task.batches(output_format='columnar')))
+ out = batch.inputs['graph']
+ events = out.nodes['events']
+ assert set(events.ids.to_pylist()) == {1, 2, 4}
+ assert set(
+ events.features['value'].to_pylist()) == {2.0, 4.0, None}
+ for edge in out.edges.values():
+ for left, right in zip(
+ edge.index['source'].to_pylist(),
+ edge.index['destination'].to_pylist(),
+ strict=True,
+ ):
+ assert (out.nodes[edge.source_type].owners[left] ==
+ out.nodes[edge.destination_type].owners[right])
+ result = events.features.to_pylist()
+ assert _files(snapshot.index_path) == before
+ return result, engine._binding.identity
+
+ expected, identity = run(snapshot, tmp_path / 'work1')
+ exports = kumo.FileConnector(root_dir=str(tmp_path / 'exports'))
+ location = snapshot.export(kumo.SnapshotOutputConfig(exports, 'saved'))
+ shutil.rmtree(data)
+ shutil.rmtree(tmp_path / 'producer')
+ kumo.configure(scratch_dir=tmp_path / 'consumer')
+ loaded = kumo.GraphSnapshot.load_from_connector(exports, path=location)
+ import diskgraph
+
+ original = diskgraph.ingest
+
+ def extension_only(*args, **kwargs):
+ assert kwargs.get('parent_index_path') == str(loaded.index_path)
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(diskgraph, 'ingest', extension_only)
+ assert run(loaded, tmp_path / 'work2') == (expected, identity)
+ assert (api.DiskGraphEngine.from_index(
+ loaded.index_path,
+ work_dir=tmp_path / 'work3')._binding.identity == identity)
+ finally:
+ kumo.configure(scratch_dir=previous)
From e4c36cbfb80dc25a86c2723db94dcb7f2269c505 Mon Sep 17 00:00:00 2001
From: Alex Porter <102990994+am-porter@users.noreply.github.com>
Date: Wed, 18 Nov 2026 15:46:51 -0800
Subject: [PATCH 2/3] Unify consumer cache paths and preserve explicit
index-local reuse
---
.../kumo-pytorch/docs/04-engines-and-data.md | 6 +-
.../kumo-pytorch/docs/api/engines.md | 2 +-
.../kumo-pytorch/src/kumo/pytorch/_storage.py | 56 ++++++++++++++
.../src/kumo/pytorch/column_stats.py | 28 +++----
.../src/kumo/pytorch/engines/diskgraph.py | 28 +++----
.../kumo-pytorch/tests/test_existing_index.py | 75 +++++++++++++++++++
6 files changed, 159 insertions(+), 36 deletions(-)
create mode 100644 packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
diff --git a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
index d23cb4453..06cb03856 100644
--- a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
+++ b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
@@ -180,7 +180,11 @@ read.
The index holds rows, key maps and relations. It holds no anchors, labels or
fitted encoders, so several contracts over the same tables can share one index.
By default, fitting keeps column statistics of encoders over stored columns in
-the index's `col_stats/`, so that directory must be writable. Supply an external
+the index's `col_stats/`, so that directory must be writable. For the ordinary
+`DiskGraphEngine(index=...)` constructor, omitting `work_dir` or explicitly setting
+it to the index path uses this same layout and reuses the same caches. Paths that
+resolve to the index, including symlinks, are equivalent. Other paths inside the
+index are rejected. Supply an external
`work_dir` to put newly computed statistics and task state there instead. Each encoder
cutoff has one directory there, with a `.col_stats.json` file per table.
A later fit at the same cutoff computes only the columns and statistics those
diff --git a/packages/experimental/kumo-pytorch/docs/api/engines.md b/packages/experimental/kumo-pytorch/docs/api/engines.md
index 34d56354f..f54fab811 100644
--- a/packages/experimental/kumo-pytorch/docs/api/engines.md
+++ b/packages/experimental/kumo-pytorch/docs/api/engines.md
@@ -332,7 +332,7 @@ step, before any data is read.
- `num_shards`: index shards.
- `ingest`: `"if_missing"` builds the index when absent; `"never"` requires an existing one. A mismatching manifest is an error, never permission to reuse.
- `encoder_cutoffs`: `encoder_cutoff` values whose column statistics an ingest computes.
-- `work_dir`: external task/cache directory; required by existing-index factories. Source-ingestion callers may omit it to retain the original index-local behavior.
+- `work_dir`: task/cache directory. Omit it or pass the index path (including an alias resolving to it) for the original index-local layout and cache reuse. Other paths must be outside the index; existing-index factories require an external directory.
- `source_versions`: optional table-to-identity mapping for source ingestion/reuse. Each entry has nonempty `source` (connector plus source identity) and `version` strings. These trusted connector versions replace file path/size/mtime identity; missing versions are never treated as equal. Omit for legacy file validation. The caller must change the version whenever source data changes: reuse does not compare Parquet contents, file sizes or mtimes in this opt-in mode.
#### `DiskGraphEngine.from_index`
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
new file mode 100644
index 000000000..d62188977
--- /dev/null
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Storage policy for index-local and external consumer state."""
+
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def normalize_work_dir(
+ index: str | Path, work_dir: str | Path | None, *, external_only: bool = False
+) -> Path | None:
+ """Use the legacy layout for an omitted or explicitly equal index path."""
+ base = Path(index).resolve()
+ work = None if work_dir is None else Path(work_dir).resolve()
+ if not external_only and (work is None or work == base):
+ return None
+ if work is None or work.is_relative_to(base):
+ raise ValueError("work_dir must be outside the base index")
+ return work
+
+
+class ColumnCache:
+ """Read immutable base caches and publish missing entries to the writable root."""
+
+ def __init__(self, index: str | Path, root: str | Path | None = None):
+ self.index = Path(index)
+ self.root = self.index if root is None else Path(root)
+ self.external = self.root.resolve() != self.index.resolve()
+
+ @classmethod
+ def for_work_dir(cls, index: str | Path, work_dir: Path | None, identity: str):
+ root = (
+ None
+ if work_dir is None
+ else work_dir / "cache" / hashlib.sha256(identity.encode()).hexdigest()
+ )
+ return cls(index, root)
+
+ def destination(self, name: str) -> Path:
+ return self.root / "col_stats" / name
+
+ def candidates(self, name: str, *, base_name: str | None = None) -> tuple[Path, ...]:
+ base = self.index / "col_stats" / (name if base_name is None else base_name)
+ return tuple(dict.fromkeys((base, self.destination(name))))
+
+ def resolve(self, name: str) -> Path:
+ return next(
+ (path for path in self.candidates(name) if path.is_file()), self.destination(name)
+ )
+
+ def reference(self, path: Path) -> str:
+ return str(path if self.external else path.relative_to(self.index / "col_stats"))
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
index fac26686a..9aa8ff610 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
@@ -20,6 +20,7 @@
import numpy as np
+from kumo.pytorch._storage import ColumnCache
from kumo.pytorch.contract.batch import (
Composite,
EntitySource,
@@ -254,9 +255,7 @@ def read_token_lengths(
if entry["file"] is None:
return entry["constant"], None
- path = Path(index, "col_stats", entry["file"])
- if not path.is_file() and cache_root is not None:
- path = Path(cache_root, "col_stats", entry["file"])
+ path = ColumnCache(index, cache_root).resolve(entry["file"])
if not path.stat().st_size:
return entry["constant"], np.frombuffer(b"", dtype=" dict:
found = {}
- for table, path in paths.items():
- for candidate in (base / path.name, path):
+ for table, name in paths.items():
+ for candidate in cache.candidates(f"{relative}/{name}", base_name=f"{cutoff}/{name}"):
if candidate.is_file():
files.add(candidate)
data = json.loads(candidate.read_text())[table]
@@ -324,7 +319,7 @@ def read() -> dict:
if missing:
request = stats_request(missing, [ticks], unit)
request["dirs"] = [relative]
- options = {"output_dir": str(destination)} if external else {}
+ options = {"output_dir": str(cache.root)} if cache.external else {}
diskgraph.cutoff_col_stats(
str(index), json.dumps(request), num_threads=num_threads, **options
)
@@ -332,10 +327,7 @@ def read() -> dict:
return ColumnStats(
{item: found[item[0]][item[1]] for item in requests},
{"value": ticks, "unit": unit},
- tuple(
- str(path if external else path.relative_to(Path(index) / "col_stats"))
- for path in sorted(files)
- ),
+ tuple(cache.reference(path) for path in sorted(files)),
)
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
index ce50935b2..0007f7ec0 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
@@ -21,6 +21,7 @@
import numpy as np
import torch
+from kumo.pytorch._storage import ColumnCache, normalize_work_dir
from kumo.pytorch.anchors import AnchorTable
from kumo.pytorch.column_stats import (
ColumnStats,
@@ -215,8 +216,10 @@ class DiskGraphEngine:
requires an existing one. A mismatching manifest is an error, never
permission to reuse.
:param encoder_cutoffs: ``encoder_cutoff`` values whose column statistics an ingest computes.
- :param work_dir: external task/cache directory; required by existing-index factories.
- Source-ingestion callers may omit it to retain the original index-local behavior.
+ :param work_dir: task/cache directory. Omit it or pass the index path (including
+ an alias resolving to it) for the original index-local layout and cache reuse.
+ Other paths must be outside the index; existing-index factories require an
+ external directory.
:param source_versions: optional table-to-identity mapping for source ingestion/reuse.
Each entry has nonempty ``source`` (connector plus source identity) and ``version``
strings. These trusted connector versions replace file path/size/mtime identity;
@@ -245,9 +248,7 @@ def __init__(
if isinstance(encoder_cutoffs, str):
raise TypeError("encoder_cutoffs is a list of cutoffs; wrap a single one in a list")
self.index = Path(index)
- self.work_dir = None if work_dir is None else Path(work_dir).resolve()
- if self.work_dir is not None and self.work_dir.is_relative_to(self.index.resolve()):
- raise ValueError("work_dir must be outside the base index")
+ self.work_dir = normalize_work_dir(self.index, work_dir)
self.source_versions = (
None
if source_versions is None
@@ -273,6 +274,7 @@ def from_index(
are hashed once; engine provenance and PyTorch manifests avoid that scan.
``work_dir`` holds task extensions and derived caches outside the base.
"""
+ work_dir = normalize_work_dir(index, work_dir, external_only=True)
result = cls(index=index, ingest="never", work_dir=work_dir, num_threads=num_threads)
result._binding = open_index(index)
return result
@@ -287,6 +289,7 @@ def from_snapshot(
unnecessary. Load exported snapshots through the existing engine loader.
No source export, base ingestion or additional metadata file is needed.
"""
+ work_dir = normalize_work_dir(snapshot.index_path, work_dir, external_only=True)
result = cls(
index=snapshot.index_path, ingest="never", work_dir=work_dir, num_threads=num_threads
)
@@ -551,11 +554,8 @@ def __init__(
json.dumps(dict(artifact.manifest), sort_keys=True).encode()
).hexdigest()
)
- self._cache_root = (
- Path(artifact.path)
- if work_dir is None
- else work_dir / "cache" / hashlib.sha256(cache_id.encode()).hexdigest()
- )
+ self._column_cache = ColumnCache.for_work_dir(artifact.path, work_dir, cache_id)
+ self._cache_root = self._column_cache.root
metadata = json.loads(Path(artifact.path, "index_metadata.json").read_text())
self._clock = metadata.get("timestamp_unit", "seconds")
self._num_threads = num_threads
@@ -1243,10 +1243,6 @@ def column_stats(self, requests: Sequence[StatRequest], cutoff: int | None) -> C
cache_root=self._cache_root,
)
- def _cache_file(self, name: str) -> Path:
- existing = Path(self._index_path, "col_stats", name)
- return existing if existing.is_file() else self._cache_root / "col_stats" / name
-
def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[str, str]:
ticks = (
None
@@ -1254,7 +1250,7 @@ def column_text(self, encoder: TableColumnEncoder, cutoff: int | None) -> tuple[
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = encoder.tokenizer_file(ticks, self._clock)
- path = self._cache_file(name)
+ path = self._column_cache.resolve(name)
if path.is_file():
try:
document = json.loads(path.read_text())
@@ -1303,7 +1299,7 @@ def token_lengths(
else self._diskgraph.convert_ticks(cutoff, "nanoseconds", self._clock, rounding="floor")
)
name = token_lengths_file(table, columns, required, ticks, self._clock)
- path = self._cache_file(name)
+ path = self._column_cache.resolve(name)
if not path.is_file():
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}")
diff --git a/packages/experimental/kumo-pytorch/tests/test_existing_index.py b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
index db17d67b0..319f1360a 100644
--- a/packages/experimental/kumo-pytorch/tests/test_existing_index.py
+++ b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
@@ -496,3 +496,78 @@ def test_connector_versions_allow_restage_but_reject_changes(indexed, tmp_path):
api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions=changed))
with pytest.raises(ValueError, match="every dataset table"):
api.prepare(spec, restaged, api.DiskGraphEngine(index=path, source_versions={}))
+
+
+@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink"])
+@pytest.mark.parametrize("explicit_first", [False, True])
+def test_explicit_index_work_dir_reuses_legacy_state(
+ indexed, tmp_path, monkeypatch, alias, explicit_first
+):
+ index, _, dataset, spec, anchors = indexed
+ if alias == "relative":
+ monkeypatch.chdir(tmp_path)
+ work = Path("index")
+ elif alias == "symlink":
+ work = tmp_path / "index-alias"
+ work.symlink_to(index, target_is_directory=True)
+ else:
+ work = index
+ encoder = TableColumnEncoder(
+ api.Text(vocabulary_size=300, min_frequency=1),
+ api.DType("string"),
+ (("events", "note"),),
+ )
+
+ def run(explicit):
+ options = {"work_dir": work} if explicit else {}
+ engine = api.DiskGraphEngine(index=index, num_threads=1, **options)
+ with api.prepare(spec, dataset, engine) as plan:
+ source = plan._source
+ stats = source.column_stats([("events", "value", "numerical")], 20 * 10**9)
+ tokenizer, name = source.column_text(encoder, 20 * 10**9)
+ lengths = source.token_lengths("events", [("note", tokenizer, 0)], [], 20 * 10**9)
+ _, values = read_token_lengths(index, {"file": lengths, "constant": 0})
+ assert all((index / "col_stats" / path).is_file() for path in stats.files)
+ assert (index / "col_stats" / name).is_file()
+ with plan.materialize(anchors) as task:
+ assert any((index / "train_tables").iterdir())
+ assert any((index / "tasks").iterdir())
+ graph = next(iter(task.batches(output_format="columnar"))).inputs["graph"]
+ assert set(graph.nodes["events"].ids.to_pylist()) == {"a", "b", "d"}
+ assert not list((index / "train_tables").iterdir())
+ assert not list((index / "tasks").iterdir())
+ return stats, tokenizer, name, lengths, values.tolist()
+
+ expected = run(explicit_first)
+ before = inventory(index)
+
+ def unexpected_compute(*args, **kwargs):
+ pytest.fail("existing in-index cache must be reused")
+
+ for function in ("cutoff_col_stats", "train_tokenizer", "token_lengths"):
+ monkeypatch.setattr(diskgraph, function, unexpected_compute)
+ assert run(not explicit_first) == expected
+ assert inventory(index) == before
+ assert not (index / "cache").exists()
+
+
+@pytest.mark.parametrize("alias", ["absolute", "relative", "symlink", "nested"])
+def test_adopted_indexes_require_external_work_dir(indexed, tmp_path, monkeypatch, alias):
+ index, _, _, _, _ = indexed
+ if alias == "relative":
+ monkeypatch.chdir(tmp_path)
+ work = Path("index")
+ elif alias == "symlink":
+ work = tmp_path / "index-alias"
+ work.symlink_to(index, target_is_directory=True)
+ else:
+ work = index if alias == "absolute" else index / "work"
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=work)
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_snapshot(
+ SimpleNamespace(index_path=index, id="test"), work_dir=work
+ )
+ if alias == "nested":
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine(index=index, work_dir=work)
From 4a68baad72ac2a8fad7bab83ce2955159311ea35 Mon Sep 17 00:00:00 2001
From: Alex Porter <102990994+am-porter@users.noreply.github.com>
Date: Wed, 18 Nov 2026 15:54:49 -0800
Subject: [PATCH 3/3] Address snapshot reuse review findings
---
.github/workflows/kumo-pytorch-diskgraph.yml | 2 +
.../kumo-pytorch/docs/04-engines-and-data.md | 17 ++-
.../kumo-pytorch/docs/api/engines.md | 2 +-
.../kumo-pytorch/src/kumo/pytorch/_storage.py | 11 +-
.../src/kumo/pytorch/column_stats.py | 6 +-
.../src/kumo/pytorch/engines/diskgraph.py | 16 ++-
.../src/kumo/pytorch/materialization.py | 10 +-
.../kumo-pytorch/tests/test_existing_index.py | 124 ++++++++++++++++++
.../kumo-engine/test/test_pytorch_snapshot.py | 13 ++
9 files changed, 190 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/kumo-pytorch-diskgraph.yml b/.github/workflows/kumo-pytorch-diskgraph.yml
index d0a5d4df5..3e312b62a 100644
--- a/.github/workflows/kumo-pytorch-diskgraph.yml
+++ b/.github/workflows/kumo-pytorch-diskgraph.yml
@@ -13,6 +13,7 @@ name: kumo-pytorch DiskGraph
on: # yamllint disable-line rule:truthy
pull_request:
paths:
+ - 'pyproject.toml'
- 'packages/experimental/kumo-pytorch/**'
- 'packages/kumo-engine/**'
- 'packages/kumo-core/**'
@@ -27,6 +28,7 @@ on: # yamllint disable-line rule:truthy
push:
branches: [main]
paths:
+ - 'pyproject.toml'
- 'packages/experimental/kumo-pytorch/**'
- 'packages/kumo-engine/**'
- 'packages/kumo-core/**'
diff --git a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
index 06cb03856..fa8f81034 100644
--- a/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
+++ b/packages/experimental/kumo-pytorch/docs/04-engines-and-data.md
@@ -184,7 +184,8 @@ the index's `col_stats/`, so that directory must be writable. For the ordinary
`DiskGraphEngine(index=...)` constructor, omitting `work_dir` or explicitly setting
it to the index path uses this same layout and reuses the same caches. Paths that
resolve to the index, including symlinks, are equivalent. Other paths inside the
-index are rejected. Supply an external
+index are rejected. External work directories and their writable roots must be
+disjoint from the base, including after resolving symlinks. Supply an external
`work_dir` to put newly computed statistics and task state there instead. Each encoder
cutoff has one directory there, with a `.col_stats.json` file per table.
A later fit at the same cutoff computes only the columns and statistics those
@@ -235,9 +236,17 @@ The base may be read-only. Task extensions, scalar-materialization scratch,
new statistics, tokenizers and token-length caches go under the external work
directory; existing immutable base caches can be read in place. Closing a task
cleans up its temporary files. Derived caches remain for reuse. No new base
-metadata file is written. Existing snapshot identity or PyTorch producer
-fingerprints survive relocation. A bare completed native index without producer
-identity is hashed once at opening, which can be expensive for large artifacts.
+metadata file is written. Fitted-state statistics provenance records absolute paths
+in external mode (including reused base files); these are diagnostic references,
+not inputs used when restoring fitted encoders. They can refer to the old location
+after relocation. External statistics share one directory per artifact and cutoff;
+the native writer locks, merges existing keys, and atomically replaces each file.
+Concurrent requests may duplicate computation but do not discard published keys.
+External storage requires a DiskGraph build supporting `cutoff_col_stats(output_dir=...)`;
+preparation checks that capability because older and newer development builds can
+share version 0.2.0. Install DiskGraph from the matching repository revision.
+Existing snapshot identity or PyTorch producer fingerprints survive relocation. A bare completed native index without producer
+identity is hashed in full on each opening, which can be expensive for large artifacts.
Opening requires the resolved `diskgraph_config.json` and the producer's
completion metadata. Engine snapshots use the native inventory/integrity check;
diff --git a/packages/experimental/kumo-pytorch/docs/api/engines.md b/packages/experimental/kumo-pytorch/docs/api/engines.md
index f54fab811..1ea55e042 100644
--- a/packages/experimental/kumo-pytorch/docs/api/engines.md
+++ b/packages/experimental/kumo-pytorch/docs/api/engines.md
@@ -353,7 +353,7 @@ Open a complete native index without its original sources or any base writes.
Read schema/relations from existing metadata. Older indexes without the
resolved native config are refused. Native edge tables and non-scalar
stored columns are not supported. Bare indexes without producer identity
-are hashed once; engine provenance and PyTorch manifests avoid that scan.
+are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan.
`work_dir` holds task extensions and derived caches outside the base.
#### `DiskGraphEngine.from_snapshot`
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
index d62188977..b8c34a8a2 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/_storage.py
@@ -18,8 +18,15 @@ def normalize_work_dir(
work = None if work_dir is None else Path(work_dir).resolve()
if not external_only and (work is None or work == base):
return None
- if work is None or work.is_relative_to(base):
+ if work is None:
raise ValueError("work_dir must be outside the base index")
+ # Validate every writable root before creating any of them, including symlinks.
+ for destination in (work, work / "tasks", work / "train_tables", work / "cache"):
+ resolved = destination.resolve()
+ if resolved.is_relative_to(base) or base.is_relative_to(resolved):
+ raise ValueError(
+ "work_dir and its writable roots must be outside and disjoint from the base index"
+ )
return work
@@ -30,6 +37,8 @@ def __init__(self, index: str | Path, root: str | Path | None = None):
self.index = Path(index)
self.root = self.index if root is None else Path(root)
self.external = self.root.resolve() != self.index.resolve()
+ if not self.external:
+ self.root = self.index
@classmethod
def for_work_dir(cls, index: str | Path, work_dir: Path | None, identity: str):
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
index 9aa8ff610..e219759e7 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/column_stats.py
@@ -292,9 +292,9 @@ def read_column_stats(
cache = ColumnCache(index, cache_root)
cutoff = _cutoff_dir(ticks, unit)
- # Isolate request sets; immutable base statistics need not be recomputed.
- recipe = hashlib.sha256(json.dumps(sorted(set(requests))).encode()).hexdigest()[:16]
- relative = cutoff + (f"/{recipe}" if cache.external else "")
+ # Native publication locks, merges existing keys, and atomically replaces files.
+ # Sharing a cutoff directory lets overlapping requests reuse prior statistics.
+ relative = cutoff
paths = {table: f"{file_name_part(table)}.col_stats.json" for table, _, _ in requests}
files = set()
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
index 0007f7ec0..7456c398f 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py
@@ -8,6 +8,7 @@
import datetime as dt
import hashlib
+import inspect
import json
import os
import time
@@ -271,7 +272,7 @@ def from_index(
Read schema/relations from existing metadata. Older indexes without the
resolved native config are refused. Native edge tables and non-scalar
stored columns are not supported. Bare indexes without producer identity
- are hashed once; engine provenance and PyTorch manifests avoid that scan.
+ are hashed in full on each open; engine provenance and PyTorch manifests avoid that scan.
``work_dir`` holds task extensions and derived caches outside the base.
"""
work_dir = normalize_work_dir(index, work_dir, external_only=True)
@@ -482,6 +483,19 @@ def open(
*,
column_stats: Sequence[StatRequest] = (),
) -> SampleSource:
+ if self.work_dir is not None:
+ import diskgraph
+
+ try:
+ inspect.signature(diskgraph.cutoff_col_stats).bind(
+ str(self.index), "{}", output_dir=str(self.work_dir)
+ )
+ except (TypeError, ValueError) as error:
+ raise RuntimeError(
+ "External work_dir requires a DiskGraph build with "
+ "cutoff_col_stats(output_dir=...) support; rebuild/install DiskGraph "
+ "from the matching kumo-engine revision"
+ ) from error
if self._binding is not None:
if dataset is not None:
raise ValueError("an existing-index engine does not accept Dataset")
diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
index 1f6bef1f3..aa79cf0ef 100644
--- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
+++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/materialization.py
@@ -93,6 +93,12 @@ def ensure_diskgraph_index(
if not dataset_name:
raise ValueError("dataset_name must be non-empty")
+ for name, source in dataset.tables.items():
+ if isinstance(source, Parquet) and source.columns:
+ raise DISKGRAPH_UNSUPPORTED["DG-COLUMN-RENAME"].error(
+ "DiskGraph materialization requires matching physical column names; "
+ f"table {name!r} declares {dict(source.columns)!r}"
+ )
output = Path(destination).resolve()
if output.exists():
artifact = _DiskGraphIndex.open(output)
@@ -104,8 +110,10 @@ def ensure_diskgraph_index(
output.parent.mkdir(parents=True, exist_ok=True)
diskgraph = _import_diskgraph()
- source_snapshot = _dataset_payload(dataset, source_versions)
physical_snapshot = _dataset_payload(dataset)
+ source_snapshot = (
+ physical_snapshot if source_versions is None else _dataset_payload(dataset, source_versions)
+ )
config, relation_mapping, source_types = _compile_diskgraph_config(
schema,
dataset,
diff --git a/packages/experimental/kumo-pytorch/tests/test_existing_index.py b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
index 319f1360a..56c947f12 100644
--- a/packages/experimental/kumo-pytorch/tests/test_existing_index.py
+++ b/packages/experimental/kumo-pytorch/tests/test_existing_index.py
@@ -571,3 +571,127 @@ def test_adopted_indexes_require_external_work_dir(indexed, tmp_path, monkeypatc
if alias == "nested":
with pytest.raises(ValueError, match="outside"):
api.DiskGraphEngine(index=index, work_dir=work)
+
+
+@pytest.mark.parametrize("root", ["tasks", "train_tables", "cache"])
+@pytest.mark.parametrize("symlink", [False, True])
+def test_external_writable_roots_cannot_overlap_base(indexed, tmp_path, root, symlink):
+ index, _, _, _, _ = indexed
+ work = tmp_path / "work"
+ work.mkdir()
+ if symlink:
+ (work / root).symlink_to(index, target_is_directory=True)
+ else:
+ moved = work / root
+ shutil.move(index, moved)
+ index = moved
+ before = inventory(index)
+ with pytest.raises(ValueError, match="outside"):
+ api.DiskGraphEngine.from_index(index, work_dir=work)
+ assert inventory(index) == before
+ assert set(p.name for p in work.iterdir()) == {root}
+
+
+def test_external_statistics_reuse_overlapping_requests(indexed, tmp_path, monkeypatch):
+ index, _, _, spec, _ = indexed
+ known = ("events", "value", "numerical")
+ missing = ("events", "value", "categorical")
+ engine = api.DiskGraphEngine.from_index(index, work_dir=tmp_path / "work", num_threads=1)
+ with api.prepare(contract=spec, engine=engine) as plan:
+ source = plan._source
+ initial = source.column_stats([known], 20 * 10**9)
+ original = diskgraph.cutoff_col_stats
+ calls = []
+
+ def compute(index_path, request_json, **options):
+ request = json.loads(request_json)
+ assert request["columns"] == {"events": {"value": ["categorical"]}}
+ calls.append(request)
+ return original(index_path, request_json, **options)
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", compute)
+ result = source.column_stats([known, missing], 20 * 10**9)
+ assert result.columns[known]["mean"] == initial.columns[known]["mean"]
+ assert result.files == initial.files
+ assert len(calls) == 1
+ assert source.column_stats([missing], 20 * 10**9).files == initial.files
+ assert len(calls) == 1
+
+
+def test_trusted_reuse_refuses_column_renames(indexed, tmp_path):
+ _, _, dataset, spec, _ = indexed
+ versions = {name: {"source": name, "version": "v1"} for name in dataset.tables}
+ index = tmp_path / "trusted"
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index, source_versions=versions)):
+ pass
+ renamed = api.Dataset(
+ {
+ name: api.Parquet(source.path, columns={"value": "renamed"})
+ if name == "events"
+ else source
+ for name, source in dataset.tables.items()
+ }
+ )
+ with pytest.raises(ValueError, match="DG-COLUMN-RENAME"):
+ api.prepare(spec, renamed, api.DiskGraphEngine(index=index, source_versions=versions))
+
+
+def test_legacy_ingest_checks_the_snapshot_recorded_in_provenance(indexed, tmp_path, monkeypatch):
+ from kumo.pytorch import materialization
+
+ _, source, dataset, spec, _ = indexed
+ original = materialization._dataset_payload
+ calls = 0
+
+ def capture(*args, **kwargs):
+ nonlocal calls
+ payload = original(*args, **kwargs)
+ calls += 1
+ if calls == 1:
+ path = source / "events.parquet"
+ table = pq.read_table(path)
+ pq.write_table(table.slice(0, 2), path)
+ return payload
+
+ monkeypatch.setattr(materialization, "_dataset_payload", capture)
+ index = tmp_path / "new-index"
+ with pytest.raises(ValueError, match="changed while DiskGraph was ingesting"):
+ api.prepare(spec, dataset, api.DiskGraphEngine(index=index, num_threads=1))
+ assert not index.exists()
+
+
+def test_external_storage_refuses_old_native_api_before_task_writes(indexed, tmp_path, monkeypatch):
+ index, _, dataset, spec, _ = indexed
+
+ def old_cutoff_col_stats(index_dir, request_json, num_threads=None):
+ pytest.fail("compatibility check must not execute statistics")
+
+ monkeypatch.setattr(diskgraph, "cutoff_col_stats", old_cutoff_col_stats)
+ work = tmp_path / "work"
+ engine = api.DiskGraphEngine.from_index(index, work_dir=work)
+ with pytest.raises(RuntimeError, match="matching kumo-engine revision"):
+ api.prepare(contract=spec, engine=engine)
+ assert not work.exists()
+ with api.prepare(spec, dataset, api.DiskGraphEngine(index=index)):
+ pass
+
+
+def test_index_local_statistics_read_each_file_once(indexed, tmp_path, monkeypatch):
+ from kumo.pytorch.column_stats import read_column_stats
+
+ index, _, _, _, _ = indexed
+ request = [("events", "value", "numerical")]
+ expected = read_column_stats(index, request, 20, "seconds")
+ alias = tmp_path / "alias"
+ alias.symlink_to(index, target_is_directory=True)
+ original = Path.read_text
+ reads = []
+
+ def read(path, *args, **kwargs):
+ if path.name.endswith(".col_stats.json"):
+ reads.append(path)
+ return original(path, *args, **kwargs)
+
+ monkeypatch.setattr(Path, "read_text", read)
+ assert read_column_stats(index, request, 20, "seconds", cache_root=alias) == expected
+ assert len(reads) == 1
diff --git a/packages/kumo-engine/test/test_pytorch_snapshot.py b/packages/kumo-engine/test/test_pytorch_snapshot.py
index 0b2de312a..6df130c83 100644
--- a/packages/kumo-engine/test/test_pytorch_snapshot.py
+++ b/packages/kumo-engine/test/test_pytorch_snapshot.py
@@ -119,6 +119,19 @@ def run(snapshot, work):
assert set(events.ids.to_pylist()) == {1, 2, 4}
assert set(
events.features['value'].to_pylist()) == {2.0, 4.0, None}
+ assert relation.name in out.edges
+ edge = out.edges[relation.name]
+ pairs = {(out.nodes[edge.source_type].ids[left].as_py(),
+ out.nodes[edge.destination_type].ids[right].as_py())
+ for left, right in zip(
+ edge.index['source'].to_pylist(),
+ edge.index['destination'].to_pylist(), strict=True)
+ }
+ expected_pairs = {(-2, 1), (-2, 2), (7, 4)}
+ if edge.source_type == 'events':
+ expected_pairs = {(event, user)
+ for user, event in expected_pairs}
+ assert pairs == expected_pairs
for edge in out.edges.values():
for left, right in zip(
edge.index['source'].to_pylist(),