From c458c2e58107b0c261399af2c906d47ccbb8b90b Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Fri, 13 Nov 2026 16:52:40 -0800 Subject: [PATCH 1/6] Avoid per-anchor reconstruction for inspectable native graphs --- .../kumo/pytorch/engines/native_anchors.py | 72 ++++- .../tests/test_inspectable_graph_batches.py | 247 ++++++++++++++++++ 2 files changed, 313 insertions(+), 6 deletions(-) create mode 100644 packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index b47f784d5..26d7fa99a 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -17,6 +17,7 @@ from collections import defaultdict from collections.abc import Iterator, Sequence from contextlib import contextmanager +from datetime import UTC from pathlib import Path from typing import Any @@ -27,6 +28,21 @@ from kumo.pytorch.materialization import _diskgraph_dtype +def _column_values(pa: Any, values: Any, dtype: Any) -> list[Any]: + """Decode a timestamp timezone once, rather than once per Arrow scalar.""" + if values.null_count == len(values): + return [None] * len(values) + if not pa.types.is_timestamp(dtype) or dtype.tz is None: + return values.cast(dtype).to_pylist() + # Let Arrow resolve its supported named/fixed-offset timezones. A scalar in + # seconds avoids an optional pandas conversion just to obtain the timezone. + tz = pa.scalar(0, type=pa.timestamp("s", tz=dtype.tz)).as_py().tzinfo + return [ + value.replace(tzinfo=UTC).astimezone(tz) if value is not None else None + for value in values.cast(pa.timestamp(dtype.unit)).to_pylist() + ] + + class NativeAnchors: """An AnchorTask backed by a TT-NG seed extension, with bounded reads. @@ -322,7 +338,9 @@ def _batch_values(self, stage, runtime): *getattr(bound, "targets", {}).values(), } metadata = { - name: columns[name].to_pylist() for name in table.metadata if name not in internal + name: _column_values(pa, columns[name], columns[name].type) + for name in table.metadata + if name not in internal } return ModelSourceBatch( sample_ids=tuple(columns[table.sample_id].to_pylist()) @@ -350,7 +368,7 @@ def _batch_values(self, stage, runtime): ), ) - def _model_batches( + def _typed_batches( self, assembler, stage, @@ -365,6 +383,7 @@ def _model_batches( rank, world_size, drop_last, + output_format, ): """One native iterator call per batch; no request objects or re-batching.""" from dataclasses import replace @@ -374,7 +393,8 @@ def _model_batches( from kumo.pytorch.engines.diskgraph import _TemporalPlanCompiler from kumo.pytorch.task import _padding_sample_ids, _surface_padding - assembler._validate_model_contract() + if output_format == "model": + assembler._validate_model_contract() source = self._source sample = assembler._model_sample orders = {} @@ -488,6 +508,8 @@ def runtime_batches(): }, ) yield _surface_padding( + # For graphs, model=True selects typed source buffers; Raw + # fields keep the same Arrow representation as inspection. assembler._collate(values, epoch=epoch, seed=seed, model=True) ) @@ -504,7 +526,7 @@ def _requests(self, stage: Any, runtime: Any) -> list[SampleRequest]: for value in values ] else: - columns[name] = values.cast(self._types[name]).to_pylist() + columns[name] = _column_values(pa, values, self._types[name]) original_positions = data["anchor_position"].to_pylist() table = self._table return [ @@ -615,6 +637,33 @@ def close(self) -> None: self._extension.cleanup() self._temporary.cleanup() + def _can_batch_inspectable_graph(self, assembler): + """Use typed buffers only when no other inspectable contract is needed.""" + from kumo.pytorch.contract.batch import Graph + + sample = assembler.sample + return ( + len(assembler._prepared_inputs) == 1 + and all( + isinstance(prepared.spec.representation, Graph) + for prepared in assembler._prepared_inputs.values() + ) + and "split" not in self._table.metadata + and not any( + ( + sample.entity_values, + sample.dated_entity_values, + sample.events, + sample.current_events, + sample.targets, + sample.candidate_sets, + sample.entity.filters, + sample.entity.available_from, + assembler.spec.targets, + ) + ) + ) + def batches( self, assembler: Any, @@ -714,12 +763,22 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): "projections": fields, } stages = self._selection(selection) - if output_format == "model" and fixed and assembler._source is self._source: + # Preserve mixed-stage ordering and validation outside graph-only contracts. + inspectable_graph = ( + output_format == "inspectable" + and len(stages) == 1 + and self._can_batch_inspectable_graph(assembler) + ) + if ( + (output_format == "model" or inspectable_graph) + and fixed + and assembler._source is self._source + ): for value in self._sample.entity_values.values(): if isinstance(value, Field) and not value.path: projections[self._sample.entity.table].append(value.column) for stage in stages: - yield from self._model_batches( + yield from self._typed_batches( assembler, stage, plan=plan, @@ -732,6 +791,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): rank=rank, world_size=world_size, drop_last=policy.drop_last if drop_last is None else drop_last, + output_format=output_format, ) return diff --git a/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py new file mode 100644 index 000000000..582e629c6 --- /dev/null +++ b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py @@ -0,0 +1,247 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inspectable native graphs retain Raw values without reconstructing requests.""" + +import sys +from datetime import date + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import torch + +from kumo.pytorch import api +from kumo.pytorch.engines.native_anchors import NativeAnchors, _column_values + + +@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) +@pytest.mark.parametrize("tz", [None, "UTC", "America/New_York", "+05:30"]) +def test_timestamp_columns_preserve_values_and_resolve_timezone_once(monkeypatch, unit, tz): + scale = {"s": 1, "ms": 1000, "us": 1000000, "ns": 1000000000}[unit] + dtype = pa.timestamp(unit, tz=tz) + # Both occurrences of the repeated DST hour, fractional precision, pre-epoch, NULL. + values = pa.chunked_array( + [[1699162200 * scale + 1, None], [1699165800 * scale + 1, -1]], type=dtype + ) + try: + expected = values.to_pylist() + except ValueError as error: + # Preserve Arrow's refusal of sub-microsecond values without pandas. + with pytest.raises(ValueError, match="Nanosecond resolution"): + _column_values(pa, values, dtype) + assert "Nanosecond resolution" in str(error) + return + attempts = [] + + class MissingPytz: + def find_spec(self, fullname, path=None, target=None): + if fullname == "pytz": + attempts.append(fullname) + raise ModuleNotFoundError("No module named 'pytz'", name="pytz") + return None + + monkeypatch.delitem(sys.modules, "pytz", raising=False) + monkeypatch.setattr(sys, "meta_path", [MissingPytz(), *sys.meta_path]) + actual = _column_values(pa, values, dtype) + assert len(attempts) <= 1 # independent of row count, including missing optional pytz + assert [None if x is None else x.isoformat() for x in actual] == [ + None if x is None else x.isoformat() for x in expected + ] + assert _column_values(pa, pa.array([], type=dtype), dtype) == [] + + +def _equal(actual, expected): + if isinstance(expected, (pa.Array, pa.ChunkedArray)): + assert actual.equals(expected) + elif isinstance(expected, torch.Tensor): + torch.testing.assert_close(actual, expected, rtol=0, atol=0, equal_nan=True) + elif isinstance(expected, dict): + assert actual.keys() == expected.keys() + for name in expected: + _equal(actual[name], expected[name]) + elif isinstance(expected, (tuple, list)): + assert len(actual) == len(expected) + for left, right in zip(actual, expected, strict=True): + _equal(left, right) + else: + assert actual == expected + + +def test_empty_timestamp_column_does_not_resolve_timezone(): + dtype = pa.timestamp("us", tz="not-a-timezone") + for values in (pa.array([], type=dtype), pa.array([None, None], type=dtype)): + assert _column_values(pa, values, dtype) == values.to_pylist() + + +@pytest.fixture +def graph_task(tmp_path, request): + pytest.importorskip("diskgraph") + variant = getattr(request, "param", "single") + ids = [2**53 + 1, 2**53 + 3, 2**53 + 5] + schema = api.DataSchema( + tables=( + api.TableSpec( + "users", + "id", + { + "id": api.DType("int64"), + "created": api.DType("timestamp", unit="s", timezone="UTC"), + }, + time_column=api.NON_TEMPORAL, + ), + api.TableSpec( + "events", + "id", + { + "id": api.DType("int64"), + "user": api.DType("int64"), + "time": api.DType("timestamp", unit="s", timezone="UTC"), + "note": api.DType("string"), + "day": api.DType("date32"), + }, + time_column="time", + ), + ), + relations=(api.RelationSpec("activity", "users", "id", "events", "user"),), + ) + users, events, anchors = ( + tmp_path / f"{name}.parquet" for name in ("users", "events", "anchors") + ) + pq.write_table( + pa.table({"id": ids, "created": pa.array([0, 0, 0], type=pa.timestamp("s", tz="UTC"))}), + users, + ) + pq.write_table( + pa.table( + { + "id": [10, 20, 30], + "user": [ids[0], ids[0], ids[1]], + "time": pa.array([10, 20, 30], type=pa.timestamp("s", tz="UTC")), + "note": ["first", None, "last"], + "day": [date(2026, 1, 1), None, date(2026, 1, 3)], + } + ), + events, + ) + sample = api.SampleSpec( + entity=api.EntitySpec( + "users", available_from="created" if variant == "availability" else None + ), + neighborhoods={ + "graph": api.NeighborhoodSpec( + (api.GraphLayer((api.GraphSample("activity", 2, strategy="recent"),)),) + ) + }, + ) + contract = api.DataLoaderSpec( + schema, + sample, + api.BatchSpec( + inputs={ + "graph": api.Input( + api.NeighborhoodSource("graph"), + api.Graph( + node_fields={ + "events": api.Fields( + tuple( + api.Encode(name, api.Raw()) for name in ("note", "day", "time") + ) + ), + } + ), + None, + ) + }, + batching=api.Batching(max_samples=2), + ), + ) + pq.write_table( + pa.table( + { + "sid": [0, 1, 2, 3, 4], + "entity": [ids[0], ids[1], ids[0], ids[2], ids[0]], + "cutoff": pa.array( + [25000001, 35000001, 15000001, 35000001, 5000001], + type=pa.timestamp("us", tz="UTC"), + ), + "split": ["a", "b", "a", "b", "a"] if variant == "mixed" else ["a"] * 5, + "metadata": [{"parts": [i, i + 1], "note": None} for i in range(5)], + "local_time": pa.array( + [1699162200000001, None, 1699165800000001, -1, 0], + type=pa.timestamp("us", tz="America/New_York"), + ), + } + ), + anchors, + ) + table = api.AnchorTable( + api.Parquet(str(anchors)), + "entity", + "cutoff", + sample_id="sid", + split="split", + metadata=("metadata", "local_time"), + ) + dataset = api.Dataset( + tables={"users": api.Parquet(str(users)), "events": api.Parquet(str(events))} + ) + with api.prepare( + contract, dataset, api.DiskGraphEngine(index=tmp_path / "index", num_threads=1) + ) as plan: + with plan.materialize(table, fit_split="a") as task: + yield task + + +@pytest.mark.parametrize("world_size", [1, 3, 7]) +@pytest.mark.parametrize("shuffle", [False, True]) +@pytest.mark.parametrize("drop_last", [False, True]) +def test_inspectable_graph_uses_native_buffers_with_exact_parity( + graph_task, monkeypatch, world_size, shuffle, drop_last +): + task = graph_task + options = dict( + world_size=world_size, + shuffle=shuffle, + drop_last=drop_last, + seed=5, + output_format="inspectable", + ) + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_can_batch_inspectable_graph", lambda *args: False) + expected = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + + def forbidden(*args, **kwargs): + raise AssertionError("inspectable graph rebuilt per-anchor requests or samples") + + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_requests", forbidden) + patch.setattr(task._assembler._source, "build", forbidden) + actual = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + for rank_batches, expected_batches in zip(actual, expected, strict=True): + assert len(rank_batches) == len(expected_batches) + for left, right in zip(rank_batches, expected_batches, strict=True): + _equal(left.inputs, right.inputs) + _equal(left.targets, right.targets) + _equal(left.metadata, right.metadata) + with pytest.raises(ValueError, match="Raw.*requires inspectable"): + list(task.batches(output_format="model")) + + +@pytest.mark.parametrize("graph_task", ["mixed", "availability"], indirect=True) +def test_other_inspectable_contracts_keep_existing_path(graph_task, monkeypatch): + task = graph_task + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_can_batch_inspectable_graph", lambda *args: False) + expected = list(task.batches()) + + def forbidden(*args, **kwargs): + raise AssertionError("unsupported inspectable contract entered typed path") + + monkeypatch.setattr(NativeAnchors, "_typed_batches", forbidden) + actual = list(task.batches()) + assert len(actual) == len(expected) + for left, right in zip(actual, expected, strict=True): + _equal(left.inputs, right.inputs) + _equal(left.metadata, right.metadata) From 727c7c6521356b5b3f801ee2a0a5172699da6a63 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Mon, 16 Nov 2026 14:56:57 -0800 Subject: [PATCH 2/6] Expose sampled Arrow graphs for foundation-model consumers --- .../kumo-pytorch/docs/05-training-batches.md | 42 +++ .../kumo-pytorch/docs/api/README.md | 3 + .../kumo-pytorch/docs/api/batches.md | 68 ++++- .../kumo-pytorch/docs/api/lifecycle.md | 10 +- .../kumo-pytorch/src/kumo/pytorch/assembly.py | 9 +- .../kumo/pytorch/contract/batch/__init__.py | 4 + .../kumo/pytorch/contract/batch/columnar.py | 70 +++++ .../src/kumo/pytorch/contract/batch/spec.py | 6 +- .../src/kumo/pytorch/engines/diskgraph.py | 73 +++++ .../kumo/pytorch/engines/native_anchors.py | 36 ++- .../kumo-pytorch/src/kumo/pytorch/task.py | 21 +- .../kumo-pytorch/tests/api_reference.py | 3 + .../tests/test_columnar_graph_batches.py | 260 ++++++++++++++++++ 13 files changed, 594 insertions(+), 11 deletions(-) create mode 100644 packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py create mode 100644 packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py diff --git a/packages/experimental/kumo-pytorch/docs/05-training-batches.md b/packages/experimental/kumo-pytorch/docs/05-training-batches.md index d526f13ce..056b2f120 100644 --- a/packages/experimental/kumo-pytorch/docs/05-training-batches.md +++ b/packages/experimental/kumo-pytorch/docs/05-training-batches.md @@ -380,3 +380,45 @@ engine](04-engines-and-data.md#the-diskgraph-engine) for how the index is built and reused. Next: [inference and serving](06-inference-and-serving.md). + +## Arrow graphs for foundation-model payloads + +For consumers that build their own model representation, a materialized +DiskGraph task can expose sampled graph columns without first encoding them +into tensor/mask dictionaries: + +```python +# graph_task has one Graph input named "graph", explicit Raw node fields, +# and anchors materialized from an AnchorTable. +for batch in graph_task.batches(batch_size=1024, output_format="columnar"): + graph = batch.inputs["graph"] # api.ColumnarGraph + roots = graph.nodes[graph.root_type] + root_ids = roots.ids.slice(0, batch.batch_size) + for table_name, nodes in graph.nodes.items(): + features = nodes.features # pyarrow.Table; NULLs stay in validity bitmaps + example_positions = nodes.owners # indexes into batch.metadata["sample_ids"] + for relation_name, edges in graph.edges.items(): + selected_pairs = edges.index # Arrow columns: source, destination +``` + +Node rows are occurrences within an example: repeated anchors and different +cutoffs do not share graph occurrences. The root table starts with one root +per batch example, in batch order. Edge endpoints index the node tables in +**declared left-to-right orientation**, including reverse traversal and +self-relations. They contain only sampled links. The same selected pair can +appear in multiple chunks; take the union when the consumer needs unique +edges. Chunks do not identify traversal direction or discovery radius. + +Feature columns contain exactly the requested Raw fields, in declaration order +and declared types. Primary-key identity is available separately in `ids`. +Native buffers are shared where compatible; restoring declared types may +allocate. All buffers remain valid after iteration and task closure, and +`Batch.to(...)` keeps them as CPU Arrow values. + +This format currently supports one graph-only input, explicit Raw encodings, +fixed sample counts, and a selection contained in one split. It refuses +request-backed tasks, non-DiskGraph engines, other sample values, targets, +entity filters, and availability checks before iteration. Select a single +split explicitly when anchors span several splits. Shuffle, rank sharding, +padding, drop-last, and iterator resume retain their existing behavior. +Existing sampling limits, including timeless destinations, still apply. diff --git a/packages/experimental/kumo-pytorch/docs/api/README.md b/packages/experimental/kumo-pytorch/docs/api/README.md index a2ec41ebb..9b54cb972 100644 --- a/packages/experimental/kumo-pytorch/docs/api/README.md +++ b/packages/experimental/kumo-pytorch/docs/api/README.md @@ -71,6 +71,9 @@ The [guide](../README.md) explains the concepts in order. The behavior spec, wit | [`Fields`](batches.md#fields) | `kumo.pytorch.api` | Select ordered fields with explicit encoders, grouping, and field IDs. | | [`Encode`](batches.md#encode) | `kumo.pytorch.api` | Encode one sampled field, optionally translating IDs into a final space. | | [`Graph`](batches.md#graph) | `kumo.pytorch.api` | Node attributes use the same explicit field encodings as ordinary columns. | +| [`ColumnarGraph`](batches.md#columnargraph) | `kumo.pytorch.api` | CPU Arrow graph buffers emitted by `Task.batches(output_format="columnar")`. | +| [`ColumnarNodes`](batches.md#columnarnodes) | `kumo.pytorch.api` | Aligned columns for node occurrences, isolated per example. | +| [`ColumnarEdges`](batches.md#columnaredges) | `kumo.pytorch.api` | Selected edges in declared left-to-right relation orientation. | | [`Columns`](batches.md#columns) | `kumo.pytorch.api` | Keep selected encoded fields as separate typed columns. | | [`Tokens`](batches.md#tokens) | `kumo.pytorch.api` | Concatenate one row's encoded IDs in the declared field order. | | [`Composite`](batches.md#composite) | `kumo.pytorch.api` | One category ID for the ordered tuple of scalar component results. | diff --git a/packages/experimental/kumo-pytorch/docs/api/batches.md b/packages/experimental/kumo-pytorch/docs/api/batches.md index 27f24fe02..e6a59b5fc 100644 --- a/packages/experimental/kumo-pytorch/docs/api/batches.md +++ b/packages/experimental/kumo-pytorch/docs/api/batches.md @@ -267,6 +267,68 @@ Node attributes use the same explicit field encodings as ordinary columns. - `node_fields`: table name to the `Fields` (scalar column encodings of physical columns) emitted for its nodes. Field-name IDs are not supported on graph nodes. +### `ColumnarGraph` + +*class* · `kumo.pytorch.api.ColumnarGraph` + +```python +class ColumnarGraph( + root_type: str, + nodes: Mapping[str, ColumnarNodes], + edges: Mapping[str, ColumnarEdges], +) +``` + +CPU Arrow graph buffers emitted by `Task.batches(output_format="columnar")`. + +`nodes` uses table names and `edges` uses declared relation names. +The first `Batch.batch_size` rows of `nodes[root_type]` are the roots, +in batch order, including repeated entities at distinct prediction times. +Isolated roots and empty declared node/edge types are retained. + +Buffers remain valid after advancing or closing the iterator/task. Treat +them as read-only. Compatible native Arrow buffers are shared; declared +dtype restoration may allocate. `Batch.to` leaves these CPU buffers in +Arrow form. This output avoids tensor encoding and is intended for callers +that construct their own model representation, such as foundation models. + +### `ColumnarNodes` + +*class* · `kumo.pytorch.api.ColumnarNodes` + +```python +class ColumnarNodes(ids: Array | ChunkedArray, owners: Array, features: Table) +``` + +Aligned columns for node occurrences, isolated per example. + +`ids` contains primary-key values (index-local row IDs when no primary +key is declared). `owners` is a non-null uint32 or uint64 Arrow array of positions +into `Batch.metadata["sample_ids"]`. The same entity may occur under +several owners. `features` is an Arrow table containing exactly the +declared Raw fields, in declaration order and declared dtypes. Arrow +validity bitmaps retain NULLs; no separate masks or filled values are used. +With no declared fields, the table has zero columns; use `len(ids)` for +the node count. All three outputs otherwise have the same row order. + +### `ColumnarEdges` + +*class* · `kumo.pytorch.api.ColumnarEdges` + +```python +class ColumnarEdges(source_type: str, destination_type: str, index: Table) +``` + +Selected edges in declared left-to-right relation orientation. + +`index` is an Arrow table with non-null uint32 or uint64 `source` and +`destination` columns, indexing rows in the corresponding node tables. +Endpoints always belong to the same example. Only edges selected by the +sampling layers are included, never all links among returned nodes. +Reverse traversals are reoriented, including self-relations. Chunks may +repeat a pair selected in both directions; consumers needing unique edges +should take their union. Chunk order is not traversal provenance. + ### `Columns` *class* · `kumo.pytorch.api.Columns` @@ -482,8 +544,10 @@ never mixed. Separate columns sit under their field names with `values` and `valid`; token outputs use `value_ids`, `position_ids`, validity and source-event indices, with optional `field_ids` and `field_valid`. Raw types without a tensor representation stay Arrow arrays in inspectable -output. Metadata carries `sample_ids`, `entity_ids`, `as_of_time`, -`splits`, `is_padding`, request metadata, fingerprints and candidate +output. Columnar output holds `ColumnarGraph` objects with Arrow feature +tables and selected edge indexes. Metadata carries `sample_ids`, +`entity_ids`, `as_of_time`, `splits`, `is_padding`, request metadata, +fingerprints and candidate identities; it is not a route for undeclared features. #### `Batch.to` diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md index a66753428..9a2b2a6da 100644 --- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md +++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md @@ -388,7 +388,7 @@ Task.batches( world_size: int = 1, drop_last: bool | None = None, chunk_batches: int = 8, - output_format: Literal['inspectable', 'model'] = 'inspectable', + output_format: Literal['inspectable', 'model', 'columnar'] = 'inspectable', ) -> TaskBatches ``` @@ -413,6 +413,14 @@ and `world_size`; ranks are disjoint before padding. A token budget representation before fetching a batch; `"inspectable"` keeps them as Arrow arrays. Both emit the same `Batch` structure. +`"columnar"` emits `ColumnarGraph` inputs with Arrow feature tables +and selected edge indexes, without tensor encoding. It requires +materialized DiskGraph anchors, one graph input with explicit Raw +fields, fixed sample counts, and a selection contained in one split. +Other sample values, targets, entity filters and availability checks +are refused before iteration. Shuffle, sharding, padding, drop-last +and resume retain their normal semantics. + #### `Task.preview` ```python diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py index 68083651e..2133904f7 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py @@ -2186,7 +2186,9 @@ def _validate_model_contract(self) -> None: "inspectable output or an explicit encoder" ) - def _collate(self, values: Any, *, epoch: int, seed: int, model=False, requests=()) -> Batch: + def _collate( + self, values: Any, *, epoch: int, seed: int, model=False, requests=(), columnar=False + ) -> Batch: declarations = { name: value.declarations for name, value in self._prepared_inputs.items() @@ -2206,6 +2208,11 @@ def _collate(self, values: Any, *, epoch: int, seed: int, model=False, requests= source=prepared.spec.source.name, node_fields=prepared.spec.representation.node_fields, ) + if columnar: + result[name] = self._source.columnar_graph_batch( + self.sample, graph, values.runtime_batch + ) + continue if model and getattr(values, "runtime_batch", None) is not None: result[name] = self._source.sample_graph_batch( self.sample, graph, prepared.state, values.runtime_batch diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py index 1aefa2450..916e8d211 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py @@ -4,6 +4,7 @@ """Batch declarations: sources, explicit encodings, arrangements, and layouts.""" +from kumo.pytorch.contract.batch.columnar import ColumnarEdges, ColumnarGraph, ColumnarNodes from kumo.pytorch.contract.batch.encodings import ( Buckets, Raw, @@ -55,6 +56,9 @@ "Buckets", "CandidateSource", "CandidateTarget", + "ColumnarEdges", + "ColumnarGraph", + "ColumnarNodes", "Columns", "Composite", "Encode", diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py new file mode 100644 index 000000000..3b488f36c --- /dev/null +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Arrow graph outputs for consumers that assemble their own model payloads.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import pyarrow as pa + + +@dataclass(frozen=True) +class ColumnarNodes: + """Aligned columns for node occurrences, isolated per example. + + ``ids`` contains primary-key values (index-local row IDs when no primary + key is declared). ``owners`` is a non-null uint32 or uint64 Arrow array of positions + into ``Batch.metadata["sample_ids"]``. The same entity may occur under + several owners. ``features`` is an Arrow table containing exactly the + declared Raw fields, in declaration order and declared dtypes. Arrow + validity bitmaps retain NULLs; no separate masks or filled values are used. + With no declared fields, the table has zero columns; use ``len(ids)`` for + the node count. All three outputs otherwise have the same row order. + """ + + ids: pa.Array | pa.ChunkedArray + owners: pa.Array + features: pa.Table + + +@dataclass(frozen=True) +class ColumnarEdges: + """Selected edges in declared left-to-right relation orientation. + + ``index`` is an Arrow table with non-null uint32 or uint64 ``source`` and + ``destination`` columns, indexing rows in the corresponding node tables. + Endpoints always belong to the same example. Only edges selected by the + sampling layers are included, never all links among returned nodes. + Reverse traversals are reoriented, including self-relations. Chunks may + repeat a pair selected in both directions; consumers needing unique edges + should take their union. Chunk order is not traversal provenance. + """ + + source_type: str + destination_type: str + index: pa.Table + + +@dataclass(frozen=True) +class ColumnarGraph: + """CPU Arrow graph buffers emitted by ``Task.batches(output_format="columnar")``. + + ``nodes`` uses table names and ``edges`` uses declared relation names. + The first ``Batch.batch_size`` rows of ``nodes[root_type]`` are the roots, + in batch order, including repeated entities at distinct prediction times. + Isolated roots and empty declared node/edge types are retained. + + Buffers remain valid after advancing or closing the iterator/task. Treat + them as read-only. Compatible native Arrow buffers are shared; declared + dtype restoration may allocate. ``Batch.to`` leaves these CPU buffers in + Arrow form. This output avoids tensor encoding and is intended for callers + that construct their own model representation, such as foundation models. + """ + + root_type: str + nodes: Mapping[str, ColumnarNodes] + edges: Mapping[str, ColumnarEdges] diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py index dac1b125a..18ffcfce0 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py @@ -346,8 +346,10 @@ class Batch: and ``valid``; token outputs use ``value_ids``, ``position_ids``, validity and source-event indices, with optional ``field_ids`` and ``field_valid``. Raw types without a tensor representation stay Arrow arrays in inspectable - output. Metadata carries ``sample_ids``, ``entity_ids``, ``as_of_time``, - ``splits``, ``is_padding``, request metadata, fingerprints and candidate + output. Columnar output holds `ColumnarGraph` objects with Arrow feature + tables and selected edge indexes. Metadata carries ``sample_ids``, + ``entity_ids``, ``as_of_time``, ``splits``, ``is_padding``, request metadata, + fingerprints and candidate identities; it is not a route for undeclared features. """ diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py index f1c7e6a04..6ea72b527 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py @@ -1287,6 +1287,79 @@ def sample_graph_batch(self, sample, spec, state, runtime): finally: self._runtime_batch = previous + def columnar_graph_batch(self, sample, spec, runtime): + """Expose sampled Arrow buffers without per-node Python or tensor assembly.""" + from kumo.pytorch.contract.batch import ColumnarEdges, ColumnarGraph, ColumnarNodes + + pa = self._pa + graph = runtime["graphs"]["main"] + neighborhood = sample.neighborhoods[spec.source] + _, physical_relations = self._sampling_plan(neighborhood) + relations = { + traversal.relation for layer in neighborhood.layers for traversal in layer.samples + } + tables = {sample.entity.table, *spec.node_fields} + for name in relations: + relation = self._relations[name] + tables.update((relation.left_table, relation.right_table)) + nodes = {} + for table in sorted(tables): + ids = graph["node_ids"].get(table, pa.array([], type=pa.uint64())) + owners = graph["batch_dict"].get(table, pa.array([], type=pa.uint64())) + fields = spec.node_fields.get(table) + names = () if fields is None else tuple(use.field for use in fields.fields) + definition = self._tables[table] + requested = tuple( + dict.fromkeys( + (*(() if definition.primary_key is None else (definition.primary_key,)), *names) + ) + ) + raw = runtime["raw_features"].get(table) + if len(owners) != len(ids): + raise RuntimeError(f"native columnar node/owner alignment mismatch: {table}") + if len(ids) and requested: + if raw is None: + raise RuntimeError(f"missing native columnar features for {table!r}") + data = pa.Table.from_batches(raw["record_batches"]) + if len(data) != len(ids) or len(owners) != len(ids): + raise RuntimeError(f"native columnar node/feature alignment mismatch: {table}") + columns = { + name: self._typed_array(data[name], definition.columns[name]) + for name in requested + } + else: + columns = { + name: pa.array([], type=definition.columns[name].to_arrow()) + for name in requested + } + nodes[table] = ColumnarNodes( + ids=columns[definition.primary_key] if definition.primary_key else ids, + owners=owners, + features=pa.table({name: columns[name] for name in names}), + ) + parts = defaultdict(list) + for edge in graph["edge_index_dict"]: + physical = edge["edge_type"]["rel"] + if physical not in physical_relations: + continue + name = physical_relations[physical] + left, right = edge["src"], edge["dst"] + if physical != self._relation_mapping[name]["forward"]: + left, right = right, left + parts[name].append(pa.record_batch([left, right], names=["source", "destination"])) + empty = pa.schema([("source", pa.uint64()), ("destination", pa.uint64())]) + edges = { + name: ColumnarEdges( + self._relations[name].left_table, + self._relations[name].right_table, + pa.Table.from_batches(parts[name]) + if parts[name] + else pa.Table.from_batches([], schema=empty), + ) + for name in sorted(relations) + } + return ColumnarGraph(sample.entity.table, nodes, edges) + def build_model_batch(self, spec, runtime, anchors, compiled): """Adapt iterator buffers; history work has finished before Python resumes.""" if self._sampler is None: diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index 1eedb5d92..7520c01f3 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -901,6 +901,17 @@ def runtime_batches(): for meta, flag in zip(anchors.request_metadata, flags, strict=True) ), ) + if output_format == "columnar": + yield _surface_padding( + assembler._collate( + replace(anchors, runtime_batch=runtime), + epoch=epoch, + seed=seed, + model=True, + columnar=True, + ) + ) + continue with runtime_features(source, runtime): values = source.build_model_batch(sample, runtime, anchors, compiled) values = replace( @@ -1043,6 +1054,27 @@ def close(self) -> None: self._event_costs = {} self._temporary.cleanup() + def validate_columnar(self, assembler, selection): + """Refuse unsupported contracts before constructing a native iterator.""" + from kumo.pytorch.contract.batch import Raw + + policy = assembler.spec.batching + if ( + not self._can_batch_inspectable_graph(assembler) + or policy.max_tokens is not None + or policy.bucket_by_length + or len(self._selection(selection)) > 1 + or assembler._source is not self._source + ): + raise ValueError( + "columnar output requires one graph-only input, fixed-size batching, " + "one selected split, and no targets, entity filters or availability" + ) + for prepared in assembler._prepared_inputs.values(): + for fields in prepared.spec.representation.node_fields.values(): + if any(not isinstance(use.encoding, Raw) for use in fields.fields): + raise ValueError("columnar graph fields require explicit Raw encodings") + def _can_batch_inspectable_graph(self, assembler): """Use typed buffers only when no other inspectable contract is needed.""" from kumo.pytorch.contract.batch import Graph @@ -1097,6 +1129,8 @@ def batches( from kumo.pytorch.contract.sample import Field from kumo.pytorch.task import _mark_padding, _surface_padding + if output_format == "columnar": + self.validate_columnar(assembler, selection) source = self._source policy = assembler.spec.batching fixed = policy.max_tokens is None and not policy.bucket_by_length @@ -1176,7 +1210,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): and self._can_batch_inspectable_graph(assembler) ) if ( - (output_format == "model" or inspectable_graph) + (output_format in {"model", "columnar"} or inspectable_graph) and (fixed or output_format == "model" and world_size == 1 and not neighborhoods) and assembler._source is self._source ): diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py index a4b27c564..b28542d67 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py @@ -209,7 +209,7 @@ def batches( world_size: int = 1, drop_last: bool | None = None, chunk_batches: int = 8, - output_format: Literal["inspectable", "model"] = "inspectable", + output_format: Literal["inspectable", "model", "columnar"] = "inspectable", ) -> TaskBatches: """Emit batches over this task's anchors, or over requests built in code. @@ -231,6 +231,14 @@ def batches( ``output_format="model"`` refuses raw types without a tensor representation before fetching a batch; ``"inspectable"`` keeps them as Arrow arrays. Both emit the same `Batch` structure. + + ``"columnar"`` emits `ColumnarGraph` inputs with Arrow feature tables + and selected edge indexes, without tensor encoding. It requires + materialized DiskGraph anchors, one graph input with explicit Raw + fields, fixed sample counts, and a selection contained in one split. + Other sample values, targets, entity filters and availability checks + are refused before iteration. Shuffle, sharding, padding, drop-last + and resume retain their normal semantics. """ if requests is not None and split is not None: @@ -264,8 +272,13 @@ def batches( "chunk_batches": chunk_batches, "output_format": output_format, } - if output_format not in {"inspectable", "model"}: - raise ValueError("output_format must be 'inspectable' or 'model'") + if output_format not in {"inspectable", "model", "columnar"}: + raise ValueError("output_format must be 'inspectable', 'model', or 'columnar'") + if output_format == "columnar": + validate = getattr(self._anchors, "validate_columnar", None) + if requests is not None or not callable(validate): + raise ValueError("columnar output requires materialized DiskGraph anchors") + validate(self._assembler, selection) if requests is None: self._anchors.count(selection) # an unknown split is refused now, not mid-iteration max_samples = ( @@ -322,7 +335,7 @@ def _anchor_batches( rank: int, world_size: int, drop_last: bool | None, - output_format: Literal["inspectable", "model"] = "inspectable", + output_format: Literal["inspectable", "model", "columnar"] = "inspectable", ) -> Iterator[Batch]: anchors = self._anchors view = getattr(anchors, "iteration_anchors", None) diff --git a/packages/experimental/kumo-pytorch/tests/api_reference.py b/packages/experimental/kumo-pytorch/tests/api_reference.py index a4c674667..5616d7c3b 100644 --- a/packages/experimental/kumo-pytorch/tests/api_reference.py +++ b/packages/experimental/kumo-pytorch/tests/api_reference.py @@ -107,6 +107,9 @@ class Page: "Fields", "Encode", "Graph", + "ColumnarGraph", + "ColumnarNodes", + "ColumnarEdges", "Columns", "Tokens", "Composite", diff --git a/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py new file mode 100644 index 000000000..a8338d92c --- /dev/null +++ b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py @@ -0,0 +1,260 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Columnar graphs preserve selected occurrences and bypass tensor assembly.""" + +import pyarrow as pa +import pyarrow.compute as pc +import pytest +import torch + +from kumo.pytorch import api +from kumo.pytorch.engines.native_anchors import NativeAnchors +from tests.test_inspectable_graph_batches import _equal +from tests.test_inspectable_graph_batches import graph_task as graph_task + + +def _compare(actual, expected): + _equal(actual.metadata, expected.metadata) + _equal(actual.targets, expected.targets) + graph = actual.inputs["graph"] + old = expected.inputs["graph"] + assert isinstance(graph, api.ColumnarGraph) + assert graph.root_type == "users" + assert graph.nodes.keys() == old["nodes"].keys() + assert graph.edges.keys() == old["edges"].keys() + for name, node in graph.nodes.items(): + previous = old["nodes"][name] + ids = previous["ids"] + ids = ids.tolist() if isinstance(ids, torch.Tensor) else ids.to_pylist() + assert node.ids.to_pylist() == ids + assert node.owners.type in (pa.uint32(), pa.uint64()) + assert node.owners.to_pylist() == previous["batch"].tolist() + assert node.features.column_names == list(previous["features"]) + for field in node.features.column_names: + values = previous["features"][field]["values"] + if isinstance(values, torch.Tensor): + values = pa.array(values.numpy()) + valid = pa.array(previous["features"][field]["valid"].numpy()) + restored = pc.if_else(valid, values, None) + assert node.features[field].combine_chunks().equals(restored) + for name, edge in graph.edges.items(): + previous = old["edges"][name] + assert edge.source_type == previous["source_type"] + assert edge.destination_type == previous["destination_type"] + pairs = set(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + assert pairs == set(map(tuple, previous["index"].numpy().T)) + for left, right in pairs: + assert ( + graph.nodes[edge.source_type].owners[left].as_py() + == graph.nodes[edge.destination_type].owners[right].as_py() + ) + + +@pytest.mark.parametrize("world_size", [1, 3, 7]) +@pytest.mark.parametrize("shuffle", [False, True]) +@pytest.mark.parametrize("drop_last", [False, True]) +def test_columnar_parity_and_lifetime(graph_task, monkeypatch, world_size, shuffle, drop_last): + task = graph_task + options = dict(world_size=world_size, shuffle=shuffle, drop_last=drop_last, seed=5) + expected = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + + def forbidden(*args, **kwargs): + raise AssertionError("columnar output rebuilt requests or encoded a public graph") + + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_requests", forbidden) + patch.setattr(task._assembler._source, "_graph_output", forbidden) + patch.setattr(task._assembler._source, "build_model_batch", forbidden) + actual = [ + list(task.batches(rank=rank, output_format="columnar", **options)) + for rank in range(world_size) + ] + # Comparing after all iterators finish also checks native buffer lifetime. + for batches, old in zip(actual, expected, strict=True): + assert len(batches) == len(old) + for batch, previous in zip(batches, old, strict=True): + _compare(batch, previous) + _compare(batch.to("cpu"), previous) + + +def test_columnar_reuses_native_buffers(graph_task, monkeypatch): + source = graph_task._assembler._source + original = source.columnar_graph_batch + captured = [] + + def capture(sample, spec, runtime): + captured.append(runtime) + return original(sample, spec, runtime) + + monkeypatch.setattr(source, "columnar_graph_batch", capture) + batches = list(graph_task.batches(output_format="columnar")) + for batch, runtime in zip(batches, captured, strict=True): + graph = batch.inputs["graph"] + native_edge_buffers = { + values.buffers()[1].address + for edge in runtime["graphs"]["main"]["edge_index_dict"] + for values in (edge["src"], edge["dst"]) + if len(values) + } + for edge in graph.edges.values(): + for column in edge.index.columns: + for chunk in column.chunks: + if len(chunk): + assert chunk.buffers()[1].address in native_edge_buffers + for name, node in graph.nodes.items(): + if not len(node.ids): + continue + assert ( + node.owners.buffers()[1].address + == runtime["graphs"]["main"]["batch_dict"][name].buffers()[1].address + ) + if name == "events": + raw = pa.Table.from_batches(runtime["raw_features"][name]["record_batches"]) + for field in ("note",): + assert ( + node.features[field].chunk(0).buffers()[1].address + == raw[field].chunk(0).buffers()[1].address + ) + + +@pytest.mark.parametrize("graph_task", ["mixed", "availability"], indirect=True) +def test_unsupported_columnar_contract_fails_before_iteration(graph_task, monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("unsupported contract opened native iterator") + + monkeypatch.setattr(NativeAnchors, "_iter", forbidden) + with pytest.raises(ValueError, match="columnar output requires"): + graph_task.batches(output_format="columnar") + + +def test_columnar_rejects_requests(graph_task): + with pytest.raises(ValueError, match="materialized DiskGraph anchors"): + graph_task.batches(requests=[], output_format="columnar") + + +def test_columnar_resume_and_closed_task_buffers(graph_task): + task = graph_task + first = task.batches(output_format="columnar", shuffle=True, seed=9) + next(first) + state = first.state_dict() + expected = list(first) + resumed = task.batches(output_format="columnar", shuffle=True, seed=9) + resumed.load_state_dict(state) + actual = list(resumed) + assert [b.metadata["sample_ids"] for b in actual] == [ + b.metadata["sample_ids"] for b in expected + ] + snapshots = [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] + task.close() + assert [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] == snapshots + + +@pytest.mark.parametrize("pk_left", [True, False]) +@pytest.mark.parametrize("directions", [("forward",), ("reverse",), ("forward", "reverse")]) +def test_columnar_self_relations_follow_declared_endpoints(tmp_path, pk_left, directions): + import pyarrow.parquet as pq + + from conformance.assertions import bound_tables + + rows = [ + {"id": 1, "parent_id": None, "at": 1}, + {"id": 2, "parent_id": 1, "at": 10}, + {"id": 3, "parent_id": 2, "at": 20}, + ] + left_key, right_key = ("id", "parent_id") if pk_left else ("parent_id", "id") + contract = api.DataLoaderSpec( + api.DataSchema( + ( + api.TableSpec( + "accounts", + "id", + { + "id": api.DType("int64"), + "parent_id": api.DType("int64"), + "at": api.DType("timestamp", unit="s", timezone="UTC"), + }, + time_column="at", + ), + ), + (api.RelationSpec("parent", "accounts", left_key, "accounts", right_key),), + ), + api.SampleSpec( + api.EntitySpec("accounts"), + neighborhoods={ + "family": api.NeighborhoodSpec( + tuple( + api.GraphLayer( + tuple( + api.GraphSample("parent", 4, direction=d, strategy="recent") + for d in directions + ) + ) + for _ in range(2) + ) + ) + }, + ), + api.BatchSpec( + { + "graph": api.Input( + api.NeighborhoodSource("family"), + api.Graph({"accounts": api.Fields((api.Encode("parent_id", api.Raw()),))}), + None, + ) + } + ), + ) + anchors = tmp_path / "anchors.parquet" + pq.write_table( + pa.table({"id": [2, 2], "cutoff": pa.array([100, 5], type=pa.timestamp("s", tz="UTC"))}), + anchors, + ) + with ( + bound_tables(contract, {"accounts": rows}, "diskgraph") as (dataset, engine), + api.prepare(contract, dataset, engine) as plan, + plan.materialize(api.AnchorTable(api.Parquet(str(anchors)), "id", "cutoff")) as task, + ): + old = next(iter(task.batches(batch_size=2))) + graph = next(iter(task.batches(batch_size=2, output_format="columnar"))).inputs["graph"] + node = graph.nodes["accounts"] + edge = graph.edges["parent"] + assert node.ids.to_pylist()[:2] == [2, 2] + pairs = list(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + assert set(pairs) == set(map(tuple, old.inputs["graph"]["edges"]["parent"]["index"].numpy().T)) + by_id = {row["id"]: row for row in rows} + for a, b in pairs: + assert by_id[node.ids[a].as_py()][left_key] == by_id[node.ids[b].as_py()][right_key] + assert node.owners[a].as_py() == node.owners[b].as_py() + + +@pytest.mark.parametrize("unsupported", ["encoding", "tokens"]) +def test_columnar_refuses_non_raw_and_token_budget(graph_task, monkeypatch, unsupported): + from dataclasses import replace + + assembler = graph_task._assembler + if unsupported == "encoding": + prepared = assembler._prepared_inputs["graph"] + graph = prepared.spec.representation + fields = dict(graph.node_fields) + fields["events"] = api.Fields((api.Encode("note", api.Vocabulary()),)) + monkeypatch.setitem( + assembler._prepared_inputs, + "graph", + replace( + prepared, + spec=replace(prepared.spec, representation=replace(graph, node_fields=fields)), + ), + ) + else: + monkeypatch.setattr( + assembler, + "spec", + replace( + assembler.spec, + batching=api.Batching(max_samples=2, max_tokens=10, length_input="graph"), + ), + ) + with pytest.raises(ValueError, match="columnar"): + graph_task.batches(output_format="columnar") From ce64904a176a489f7477a933b68345494023ce53 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Fri, 13 Nov 2026 16:52:40 -0800 Subject: [PATCH 3/6] Avoid per-anchor reconstruction for inspectable native graphs --- .../kumo/pytorch/engines/native_anchors.py | 84 +++++- .../tests/test_inspectable_graph_batches.py | 256 ++++++++++++++++++ 2 files changed, 332 insertions(+), 8 deletions(-) create mode 100644 packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index 143707bad..1eedb5d92 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -17,6 +17,7 @@ from collections import defaultdict from collections.abc import Iterator, Sequence from contextlib import contextmanager +from datetime import UTC from pathlib import Path from typing import Any @@ -27,6 +28,34 @@ from kumo.pytorch.materialization import _diskgraph_dtype +def _column_values(pa: Any, values: Any, dtype: Any) -> list[Any]: + """Decode a timestamp timezone once, rather than once per Arrow scalar.""" + if values.null_count == len(values): + return [None] * len(values) + if pa.types.is_timestamp(dtype) and dtype.unit == "ns": + # Preserve main's lossless Arrow scalars for sub-microsecond values. + # Resolve timezones column-wise for the aligned Python datetime values. + typed = values.cast(dtype) + converted = _column_values( + pa, + typed.cast(pa.timestamp("us", tz=dtype.tz), safe=False), + pa.timestamp("us", tz=dtype.tz), + ) + return [ + value if value.is_valid and value.value % 1000 else decoded + for value, decoded in zip(typed, converted, strict=True) + ] + if not pa.types.is_timestamp(dtype) or dtype.tz is None: + return _arrow_values(values.cast(dtype)) + # Let Arrow resolve its supported named/fixed-offset timezones. A scalar in + # seconds avoids an optional pandas conversion just to obtain the timezone. + tz = pa.scalar(0, type=pa.timestamp("s", tz=dtype.tz)).as_py().tzinfo + return [ + value.replace(tzinfo=UTC).astimezone(tz) if value is not None else None + for value in values.cast(pa.timestamp(dtype.unit)).to_pylist() + ] + + class NativeAnchors: """An AnchorTask backed by a TT-NG seed extension, with bounded reads. @@ -361,7 +390,9 @@ def _batch_values(self, stage, runtime): *getattr(bound, "targets", {}).values(), } metadata = { - name: _arrow_values(columns[name]) for name in table.metadata if name not in internal + name: _column_values(pa, columns[name], columns[name].type) + for name in table.metadata + if name not in internal } targets = {} for name, column in {**table.targets, **getattr(bound, "targets", {})}.items(): @@ -728,7 +759,7 @@ def always_present(value): ) return contract, token_lengths - def _model_batches( + def _typed_batches( self, assembler, stage, @@ -744,6 +775,7 @@ def _model_batches( rank, world_size, drop_last, + output_format, ): """One native iterator call per batch; no request objects or re-batching.""" from dataclasses import replace @@ -753,7 +785,8 @@ def _model_batches( from kumo.pytorch.engines.diskgraph import _TemporalPlanCompiler from kumo.pytorch.task import _padding_sample_ids, _surface_padding - assembler._validate_model_contract() + if output_format == "model": + assembler._validate_model_contract() source = self._source sample = assembler._model_sample orders = {} @@ -877,6 +910,8 @@ def runtime_batches(): }, ) yield _surface_padding( + # For graphs, model=True selects typed source buffers; Raw + # fields keep the same Arrow representation as inspection. assembler._collate(values, epoch=epoch, seed=seed, model=True) ) @@ -893,8 +928,7 @@ def _requests(self, stage: Any, runtime: Any) -> list[SampleRequest]: for value in values ] else: - typed = values.cast(self._types[name]) - columns[name] = _arrow_values(typed) + columns[name] = _column_values(pa, values, self._types[name]) original_positions = data["anchor_position"].to_pylist() table = self._table return [ @@ -1009,6 +1043,33 @@ def close(self) -> None: self._event_costs = {} self._temporary.cleanup() + def _can_batch_inspectable_graph(self, assembler): + """Use typed buffers only when no other inspectable contract is needed.""" + from kumo.pytorch.contract.batch import Graph + + sample = assembler.sample + return ( + len(assembler._prepared_inputs) == 1 + and all( + isinstance(prepared.spec.representation, Graph) + for prepared in assembler._prepared_inputs.values() + ) + and "split" not in self._table.metadata + and not any( + ( + sample.entity_values, + sample.dated_entity_values, + sample.events, + sample.current_events, + sample.targets, + sample.candidate_sets, + sample.entity.filters, + sample.entity.available_from, + assembler.spec.targets, + ) + ) + ) + def batches( self, assembler: Any, @@ -1108,16 +1169,22 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): "projections": fields, } stages = self._selection(selection) + # Preserve mixed-stage ordering and validation outside graph-only contracts. + inspectable_graph = ( + output_format == "inspectable" + and len(stages) == 1 + and self._can_batch_inspectable_graph(assembler) + ) if ( - output_format == "model" - and (fixed or world_size == 1 and not neighborhoods) + (output_format == "model" or inspectable_graph) + and (fixed or output_format == "model" and world_size == 1 and not neighborhoods) and assembler._source is self._source ): for value in self._sample.entity_values.values(): if isinstance(value, Field) and not value.path: projections[self._sample.entity.table].append(value.column) for stage in stages: - yield from self._model_batches( + yield from self._typed_batches( assembler, stage, plan=plan, @@ -1131,6 +1198,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): rank=rank, world_size=world_size, drop_last=policy.drop_last if drop_last is None else drop_last, + output_format=output_format, ) return diff --git a/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py new file mode 100644 index 000000000..abe2e0de4 --- /dev/null +++ b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py @@ -0,0 +1,256 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inspectable native graphs retain Raw values without reconstructing requests.""" + +import sys +from datetime import date + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +import torch + +from kumo.pytorch import api +from kumo.pytorch.contract.dtypes import _arrow_values +from kumo.pytorch.engines.native_anchors import NativeAnchors, _column_values + + +@pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) +@pytest.mark.parametrize("tz", [None, "UTC", "America/New_York", "+05:30"]) +def test_timestamp_columns_preserve_values_and_resolve_timezone_once(monkeypatch, unit, tz): + scale = {"s": 1, "ms": 1000, "us": 1000000, "ns": 1000000000}[unit] + dtype = pa.timestamp(unit, tz=tz) + # Both occurrences of the repeated DST hour, fractional precision, pre-epoch, NULL. + values = pa.chunked_array( + [[1699162200 * scale + 1, None], [1699165800 * scale, -1, -1000, 0]], type=dtype + ) + expected = _arrow_values(values) + attempts = [] + + class MissingPytz: + def find_spec(self, fullname, path=None, target=None): + if fullname == "pytz": + attempts.append(fullname) + raise ModuleNotFoundError("No module named 'pytz'", name="pytz") + return None + + monkeypatch.delitem(sys.modules, "pytz", raising=False) + monkeypatch.setattr(sys, "meta_path", [MissingPytz(), *sys.meta_path]) + actual = _column_values(pa, values, dtype) + assert len(attempts) <= 1 # independent of row count, including missing optional pytz + _equal(actual, expected) + assert _column_values(pa, pa.array([], type=dtype), dtype) == [] + + +@pytest.mark.parametrize("nested", [False, True]) +def test_nanosecond_columns_preserve_main_scalar_contract_without_pandas(monkeypatch, nested): + monkeypatch.setitem(sys.modules, "pandas", None) + dtype = pa.timestamp("ns", tz="UTC") + if nested: + dtype = pa.list_(dtype) + values = pa.array([[1, -1, 1000, None], None, []], type=dtype) + else: + values = pa.array([1, -1, 1000, None], type=dtype) + expected = _arrow_values(values) + actual = _column_values(pa, values, dtype) + _equal(actual, expected) + fractional = actual[0][0] if nested else actual[0] + assert isinstance(fractional, pa.TimestampScalar) + assert fractional.value == 1 + + +def _equal(actual, expected): + if isinstance(expected, (pa.Array, pa.ChunkedArray)): + assert actual.equals(expected) + elif isinstance(expected, torch.Tensor): + torch.testing.assert_close(actual, expected, rtol=0, atol=0, equal_nan=True) + elif isinstance(expected, dict): + assert actual.keys() == expected.keys() + for name in expected: + _equal(actual[name], expected[name]) + elif isinstance(expected, (tuple, list)): + assert len(actual) == len(expected) + for left, right in zip(actual, expected, strict=True): + _equal(left, right) + else: + assert actual == expected + + +def test_empty_timestamp_column_does_not_resolve_timezone(): + dtype = pa.timestamp("us", tz="not-a-timezone") + for values in (pa.array([], type=dtype), pa.array([None, None], type=dtype)): + assert _column_values(pa, values, dtype) == values.to_pylist() + + +@pytest.fixture +def graph_task(tmp_path, request): + pytest.importorskip("diskgraph") + variant = getattr(request, "param", "single") + ids = [2**53 + 1, 2**53 + 3, 2**53 + 5] + schema = api.DataSchema( + tables=( + api.TableSpec( + "users", + "id", + { + "id": api.DType("int64"), + "created": api.DType("timestamp", unit="s", timezone="UTC"), + }, + time_column=api.NON_TEMPORAL, + ), + api.TableSpec( + "events", + "id", + { + "id": api.DType("int64"), + "user": api.DType("int64"), + "time": api.DType("timestamp", unit="s", timezone="UTC"), + "note": api.DType("string"), + "day": api.DType("date32"), + }, + time_column="time", + ), + ), + relations=(api.RelationSpec("activity", "users", "id", "events", "user"),), + ) + users, events, anchors = ( + tmp_path / f"{name}.parquet" for name in ("users", "events", "anchors") + ) + pq.write_table( + pa.table({"id": ids, "created": pa.array([0, 0, 0], type=pa.timestamp("s", tz="UTC"))}), + users, + ) + pq.write_table( + pa.table( + { + "id": [10, 20, 30], + "user": [ids[0], ids[0], ids[1]], + "time": pa.array([10, 20, 30], type=pa.timestamp("s", tz="UTC")), + "note": ["first", None, "last"], + "day": [date(2026, 1, 1), None, date(2026, 1, 3)], + } + ), + events, + ) + sample = api.SampleSpec( + entity=api.EntitySpec( + "users", available_from="created" if variant == "availability" else None + ), + neighborhoods={ + "graph": api.NeighborhoodSpec( + (api.GraphLayer((api.GraphSample("activity", 2, strategy="recent"),)),) + ) + }, + ) + contract = api.DataLoaderSpec( + schema, + sample, + api.BatchSpec( + inputs={ + "graph": api.Input( + api.NeighborhoodSource("graph"), + api.Graph( + node_fields={ + "events": api.Fields( + tuple( + api.Encode(name, api.Raw()) for name in ("note", "day", "time") + ) + ), + } + ), + None, + ) + }, + batching=api.Batching(max_samples=2), + ), + ) + pq.write_table( + pa.table( + { + "sid": [0, 1, 2, 3, 4], + "entity": [ids[0], ids[1], ids[0], ids[2], ids[0]], + "cutoff": pa.array( + [25000001, 35000001, 15000001, 35000001, 5000001], + type=pa.timestamp("us", tz="UTC"), + ), + "split": ["a", "b", "a", "b", "a"] if variant == "mixed" else ["a"] * 5, + "metadata": [{"parts": [i, i + 1], "note": None} for i in range(5)], + "local_time": pa.array( + [1699162200000001, None, 1699165800000001, -1, 0], + type=pa.timestamp("us", tz="America/New_York"), + ), + } + ), + anchors, + ) + table = api.AnchorTable( + api.Parquet(str(anchors)), + "entity", + "cutoff", + sample_id="sid", + split="split", + metadata=("metadata", "local_time"), + ) + dataset = api.Dataset( + tables={"users": api.Parquet(str(users)), "events": api.Parquet(str(events))} + ) + with api.prepare( + contract, dataset, api.DiskGraphEngine(index=tmp_path / "index", num_threads=1) + ) as plan: + with plan.materialize(table, fit_split="a") as task: + yield task + + +@pytest.mark.parametrize("world_size", [1, 3, 7]) +@pytest.mark.parametrize("shuffle", [False, True]) +@pytest.mark.parametrize("drop_last", [False, True]) +def test_inspectable_graph_uses_native_buffers_with_exact_parity( + graph_task, monkeypatch, world_size, shuffle, drop_last +): + task = graph_task + options = dict( + world_size=world_size, + shuffle=shuffle, + drop_last=drop_last, + seed=5, + output_format="inspectable", + ) + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_can_batch_inspectable_graph", lambda *args: False) + expected = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + + def forbidden(*args, **kwargs): + raise AssertionError("inspectable graph rebuilt per-anchor requests or samples") + + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_requests", forbidden) + patch.setattr(task._assembler._source, "build", forbidden) + actual = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + for rank_batches, expected_batches in zip(actual, expected, strict=True): + assert len(rank_batches) == len(expected_batches) + for left, right in zip(rank_batches, expected_batches, strict=True): + _equal(left.inputs, right.inputs) + _equal(left.targets, right.targets) + _equal(left.metadata, right.metadata) + with pytest.raises(ValueError, match="Raw.*requires inspectable"): + list(task.batches(output_format="model")) + + +@pytest.mark.parametrize("graph_task", ["mixed", "availability"], indirect=True) +def test_other_inspectable_contracts_keep_existing_path(graph_task, monkeypatch): + task = graph_task + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_can_batch_inspectable_graph", lambda *args: False) + expected = list(task.batches()) + + def forbidden(*args, **kwargs): + raise AssertionError("unsupported inspectable contract entered typed path") + + monkeypatch.setattr(NativeAnchors, "_typed_batches", forbidden) + actual = list(task.batches()) + assert len(actual) == len(expected) + for left, right in zip(actual, expected, strict=True): + _equal(left.inputs, right.inputs) + _equal(left.metadata, right.metadata) From 0fffd73bba2d35f9d7eab00c34c23df6eba3b234 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Mon, 16 Nov 2026 15:06:01 -0800 Subject: [PATCH 4/6] Fix timezone oracle under missing pytz --- .../kumo-pytorch/tests/test_inspectable_graph_batches.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py index abe2e0de4..f4b868b10 100644 --- a/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py +++ b/packages/experimental/kumo-pytorch/tests/test_inspectable_graph_batches.py @@ -26,7 +26,6 @@ def test_timestamp_columns_preserve_values_and_resolve_timezone_once(monkeypatch values = pa.chunked_array( [[1699162200 * scale + 1, None], [1699165800 * scale, -1, -1000, 0]], type=dtype ) - expected = _arrow_values(values) attempts = [] class MissingPytz: @@ -38,6 +37,8 @@ def find_spec(self, fullname, path=None, target=None): monkeypatch.delitem(sys.modules, "pytz", raising=False) monkeypatch.setattr(sys, "meta_path", [MissingPytz(), *sys.meta_path]) + expected = _arrow_values(values) + attempts.clear() actual = _column_values(pa, values, dtype) assert len(attempts) <= 1 # independent of row count, including missing optional pytz _equal(actual, expected) From b335c24d1563f7c8b8ab93e1e4037e6f2a808899 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Mon, 16 Nov 2026 20:13:27 -0800 Subject: [PATCH 5/6] Add optional stable edge deduplication to columnar batches --- .../kumo-pytorch/docs/05-training-batches.md | 8 +- .../kumo-pytorch/docs/api/batches.md | 7 +- .../kumo-pytorch/docs/api/lifecycle.md | 6 +- .../kumo-pytorch/src/kumo/pytorch/assembly.py | 15 +- .../kumo/pytorch/contract/batch/columnar.py | 7 +- .../src/kumo/pytorch/engines/diskgraph.py | 29 ++-- .../kumo/pytorch/engines/native_anchors.py | 4 + .../kumo-pytorch/src/kumo/pytorch/task.py | 14 +- .../tests/test_columnar_graph_batches.py | 131 ++++++++++++++++-- 9 files changed, 194 insertions(+), 27 deletions(-) diff --git a/packages/experimental/kumo-pytorch/docs/05-training-batches.md b/packages/experimental/kumo-pytorch/docs/05-training-batches.md index 056b2f120..7348ffad7 100644 --- a/packages/experimental/kumo-pytorch/docs/05-training-batches.md +++ b/packages/experimental/kumo-pytorch/docs/05-training-batches.md @@ -406,8 +406,12 @@ cutoffs do not share graph occurrences. The root table starts with one root per batch example, in batch order. Edge endpoints index the node tables in **declared left-to-right orientation**, including reverse traversal and self-relations. They contain only sampled links. The same selected pair can -appear in multiple chunks; take the union when the consumer needs unique -edges. Chunks do not identify traversal direction or discovery radius. +appear in multiple chunks. Consumers can normalize these pairs while building +their payload, or pass `deduplicate_edges=True` to `batches` for unique pairs +per relation in first-occurrence order. This opt-in step materializes edge +arrays and adds deduplication work; the default `False` retains native chunks. +Deduplication preserves separate examples, including repeated entity IDs. +Chunks do not identify traversal direction or discovery radius. Feature columns contain exactly the requested Raw fields, in declaration order and declared types. Primary-key identity is available separately in `ids`. diff --git a/packages/experimental/kumo-pytorch/docs/api/batches.md b/packages/experimental/kumo-pytorch/docs/api/batches.md index e6a59b5fc..ce0e227d8 100644 --- a/packages/experimental/kumo-pytorch/docs/api/batches.md +++ b/packages/experimental/kumo-pytorch/docs/api/batches.md @@ -326,8 +326,11 @@ Selected edges in declared left-to-right relation orientation. Endpoints always belong to the same example. Only edges selected by the sampling layers are included, never all links among returned nodes. Reverse traversals are reoriented, including self-relations. Chunks may -repeat a pair selected in both directions; consumers needing unique edges -should take their union. Chunk order is not traversal provenance. +repeat a pair selected in both directions. Consumers must normalize these +pairs or request `Task.batches(..., deduplicate_edges=True)` to retain +the first occurrence per relation, at the cost of edge-array allocation. +Deduplication uses node positions, preserving separate examples even when +entity IDs repeat. Chunk order is not traversal provenance. ### `Columns` diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md index 9a2b2a6da..85aefeb1d 100644 --- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md +++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md @@ -389,6 +389,7 @@ Task.batches( drop_last: bool | None = None, chunk_batches: int = 8, output_format: Literal['inspectable', 'model', 'columnar'] = 'inspectable', + deduplicate_edges: bool = False, ) -> TaskBatches ``` @@ -419,7 +420,10 @@ materialized DiskGraph anchors, one graph input with explicit Raw fields, fixed sample counts, and a selection contained in one split. Other sample values, targets, entity filters and availability checks are refused before iteration. Shuffle, sharding, padding, drop-last -and resume retain their normal semantics. +and resume retain their normal semantics. By default, selected edge pairs +may repeat across native chunks. `deduplicate_edges=True` is available +only for columnar output: it allocates edge arrays to retain the first +occurrence of each pair per relation, preserving example isolation. #### `Task.preview` diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py index 2133904f7..36fd2540a 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py @@ -2187,7 +2187,15 @@ def _validate_model_contract(self) -> None: ) def _collate( - self, values: Any, *, epoch: int, seed: int, model=False, requests=(), columnar=False + self, + values: Any, + *, + epoch: int, + seed: int, + model=False, + requests=(), + columnar=False, + deduplicate_edges=False, ) -> Batch: declarations = { name: value.declarations @@ -2210,7 +2218,10 @@ def _collate( ) if columnar: result[name] = self._source.columnar_graph_batch( - self.sample, graph, values.runtime_batch + self.sample, + graph, + values.runtime_batch, + deduplicate_edges=deduplicate_edges, ) continue if model and getattr(values, "runtime_batch", None) is not None: diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py index 3b488f36c..c97be0a3b 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py @@ -40,8 +40,11 @@ class ColumnarEdges: Endpoints always belong to the same example. Only edges selected by the sampling layers are included, never all links among returned nodes. Reverse traversals are reoriented, including self-relations. Chunks may - repeat a pair selected in both directions; consumers needing unique edges - should take their union. Chunk order is not traversal provenance. + repeat a pair selected in both directions. Consumers must normalize these + pairs or request ``Task.batches(..., deduplicate_edges=True)`` to retain + the first occurrence per relation, at the cost of edge-array allocation. + Deduplication uses node positions, preserving separate examples even when + entity IDs repeat. Chunk order is not traversal provenance. """ source_type: str diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py index 6ea72b527..c37517c78 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py @@ -1287,7 +1287,7 @@ def sample_graph_batch(self, sample, spec, state, runtime): finally: self._runtime_batch = previous - def columnar_graph_batch(self, sample, spec, runtime): + def columnar_graph_batch(self, sample, spec, runtime, *, deduplicate_edges=False): """Expose sampled Arrow buffers without per-node Python or tensor assembly.""" from kumo.pytorch.contract.batch import ColumnarEdges, ColumnarGraph, ColumnarNodes @@ -1348,16 +1348,29 @@ def columnar_graph_batch(self, sample, spec, runtime): left, right = right, left parts[name].append(pa.record_batch([left, right], names=["source", "destination"])) empty = pa.schema([("source", pa.uint64()), ("destination", pa.uint64())]) - edges = { - name: ColumnarEdges( - self._relations[name].left_table, - self._relations[name].right_table, + edges = {} + for name in sorted(relations): + index = ( pa.Table.from_batches(parts[name]) if parts[name] - else pa.Table.from_batches([], schema=empty), + else pa.Table.from_batches([], schema=empty) + ) + if deduplicate_edges and len(index) > 1: + # Opt-in materialization; the default retains native edge chunks. + pairs = _first_occurrences( + np.stack([column.to_numpy() for column in index.columns]) + ) + index = pa.table( + { + field.name: pa.array(values, type=field.type) + for field, values in zip(index.schema, pairs, strict=True) + } + ) + edges[name] = ColumnarEdges( + self._relations[name].left_table, + self._relations[name].right_table, + index, ) - for name in sorted(relations) - } return ColumnarGraph(sample.entity.table, nodes, edges) def build_model_batch(self, spec, runtime, anchors, compiled): diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index 7520c01f3..56df0aa2c 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -776,6 +776,7 @@ def _typed_batches( world_size, drop_last, output_format, + deduplicate_edges=False, ): """One native iterator call per batch; no request objects or re-batching.""" from dataclasses import replace @@ -909,6 +910,7 @@ def runtime_batches(): seed=seed, model=True, columnar=True, + deduplicate_edges=deduplicate_edges, ) ) continue @@ -1116,6 +1118,7 @@ def batches( world_size: int, drop_last: bool | None, output_format: str, + deduplicate_edges: bool = False, ): """Read native anchor batches and retain the existing fitted assembler. @@ -1233,6 +1236,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): world_size=world_size, drop_last=policy.drop_last if drop_last is None else drop_last, output_format=output_format, + deduplicate_edges=deduplicate_edges, ) return diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py index b28542d67..b9d4a5689 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py @@ -210,6 +210,7 @@ def batches( drop_last: bool | None = None, chunk_batches: int = 8, output_format: Literal["inspectable", "model", "columnar"] = "inspectable", + deduplicate_edges: bool = False, ) -> TaskBatches: """Emit batches over this task's anchors, or over requests built in code. @@ -238,7 +239,10 @@ def batches( fields, fixed sample counts, and a selection contained in one split. Other sample values, targets, entity filters and availability checks are refused before iteration. Shuffle, sharding, padding, drop-last - and resume retain their normal semantics. + and resume retain their normal semantics. By default, selected edge pairs + may repeat across native chunks. ``deduplicate_edges=True`` is available + only for columnar output: it allocates edge arrays to retain the first + occurrence of each pair per relation, preserving example isolation. """ if requests is not None and split is not None: @@ -274,7 +278,12 @@ def batches( } if output_format not in {"inspectable", "model", "columnar"}: raise ValueError("output_format must be 'inspectable', 'model', or 'columnar'") + if not isinstance(deduplicate_edges, bool): + raise ValueError("deduplicate_edges must be a bool") + if deduplicate_edges and output_format != "columnar": + raise ValueError("deduplicate_edges requires columnar output") if output_format == "columnar": + config["deduplicate_edges"] = deduplicate_edges validate = getattr(self._anchors, "validate_columnar", None) if requests is not None or not callable(validate): raise ValueError("columnar output requires materialized DiskGraph anchors") @@ -302,6 +311,7 @@ def produce() -> Iterator[Batch]: world_size=world_size, drop_last=drop_last, output_format=output_format, + deduplicate_edges=deduplicate_edges, ) else: @@ -336,6 +346,7 @@ def _anchor_batches( world_size: int, drop_last: bool | None, output_format: Literal["inspectable", "model", "columnar"] = "inspectable", + deduplicate_edges: bool = False, ) -> Iterator[Batch]: anchors = self._anchors view = getattr(anchors, "iteration_anchors", None) @@ -355,6 +366,7 @@ def _anchor_batches( world_size=world_size, drop_last=drop_last, output_format=output_format, + **({"deduplicate_edges": deduplicate_edges} if output_format == "columnar" else {}), ) return positions = padded_positions( diff --git a/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py index a8338d92c..e34fd3a2e 100644 --- a/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py +++ b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py @@ -52,10 +52,13 @@ def _compare(actual, expected): ) +@pytest.mark.parametrize("deduplicate_edges", [False, True]) @pytest.mark.parametrize("world_size", [1, 3, 7]) @pytest.mark.parametrize("shuffle", [False, True]) @pytest.mark.parametrize("drop_last", [False, True]) -def test_columnar_parity_and_lifetime(graph_task, monkeypatch, world_size, shuffle, drop_last): +def test_columnar_parity_and_lifetime( + graph_task, monkeypatch, world_size, shuffle, drop_last, deduplicate_edges +): task = graph_task options = dict(world_size=world_size, shuffle=shuffle, drop_last=drop_last, seed=5) expected = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] @@ -68,7 +71,14 @@ def forbidden(*args, **kwargs): patch.setattr(task._assembler._source, "_graph_output", forbidden) patch.setattr(task._assembler._source, "build_model_batch", forbidden) actual = [ - list(task.batches(rank=rank, output_format="columnar", **options)) + list( + task.batches( + rank=rank, + output_format="columnar", + deduplicate_edges=deduplicate_edges, + **options, + ) + ) for rank in range(world_size) ] # Comparing after all iterators finish also checks native buffer lifetime. @@ -77,6 +87,10 @@ def forbidden(*args, **kwargs): for batch, previous in zip(batches, old, strict=True): _compare(batch, previous) _compare(batch.to("cpu"), previous) + if deduplicate_edges: + for edge in batch.inputs["graph"].edges.values(): + pairs = _pairs(edge) + assert len(pairs) == len(set(pairs)) def test_columnar_reuses_native_buffers(graph_task, monkeypatch): @@ -84,10 +98,14 @@ def test_columnar_reuses_native_buffers(graph_task, monkeypatch): original = source.columnar_graph_batch captured = [] - def capture(sample, spec, runtime): + def capture(sample, spec, runtime, **kwargs): captured.append(runtime) - return original(sample, spec, runtime) + return original(sample, spec, runtime, **kwargs) + + def forbidden(*args, **kwargs): + raise AssertionError("default columnar output attempted deduplication") + monkeypatch.setattr("kumo.pytorch.engines.diskgraph._first_occurrences", forbidden) monkeypatch.setattr(source, "columnar_graph_batch", capture) batches = list(graph_task.batches(output_format="columnar")) for batch, runtime in zip(batches, captured, strict=True): @@ -134,26 +152,42 @@ def test_columnar_rejects_requests(graph_task): graph_task.batches(requests=[], output_format="columnar") -def test_columnar_resume_and_closed_task_buffers(graph_task): +@pytest.mark.parametrize("deduplicate_edges", [False, True]) +def test_columnar_resume_and_closed_task_buffers(graph_task, deduplicate_edges): task = graph_task - first = task.batches(output_format="columnar", shuffle=True, seed=9) + first = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=deduplicate_edges + ) next(first) state = first.state_dict() + changed = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=not deduplicate_edges + ) + with pytest.raises(ValueError, match="different batch configuration"): + changed.load_state_dict(state) expected = list(first) - resumed = task.batches(output_format="columnar", shuffle=True, seed=9) + resumed = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=deduplicate_edges + ) resumed.load_state_dict(state) actual = list(resumed) assert [b.metadata["sample_ids"] for b in actual] == [ b.metadata["sample_ids"] for b in expected ] + for batch, previous in zip(actual, expected, strict=True): + for name, edge in batch.inputs["graph"].edges.items(): + assert edge.index.equals(previous.inputs["graph"].edges[name].index) snapshots = [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] task.close() assert [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] == snapshots +@pytest.mark.parametrize("deduplicate_edges", [False, True]) @pytest.mark.parametrize("pk_left", [True, False]) @pytest.mark.parametrize("directions", [("forward",), ("reverse",), ("forward", "reverse")]) -def test_columnar_self_relations_follow_declared_endpoints(tmp_path, pk_left, directions): +def test_columnar_self_relations_follow_declared_endpoints( + tmp_path, pk_left, directions, deduplicate_edges +): import pyarrow.parquet as pq from conformance.assertions import bound_tables @@ -217,7 +251,14 @@ def test_columnar_self_relations_follow_declared_endpoints(tmp_path, pk_left, di plan.materialize(api.AnchorTable(api.Parquet(str(anchors)), "id", "cutoff")) as task, ): old = next(iter(task.batches(batch_size=2))) - graph = next(iter(task.batches(batch_size=2, output_format="columnar"))).inputs["graph"] + raw = next(task.batches(batch_size=2, output_format="columnar")).inputs["graph"] + graph = next( + task.batches( + batch_size=2, output_format="columnar", deduplicate_edges=deduplicate_edges + ) + ).inputs["graph"] + if deduplicate_edges: + assert _pairs(graph.edges["parent"]) == list(dict.fromkeys(_pairs(raw.edges["parent"]))) node = graph.nodes["accounts"] edge = graph.edges["parent"] assert node.ids.to_pylist()[:2] == [2, 2] @@ -258,3 +299,75 @@ def test_columnar_refuses_non_raw_and_token_budget(graph_task, monkeypatch, unsu ) with pytest.raises(ValueError, match="columnar"): graph_task.batches(output_format="columnar") + + +def _pairs(edge): + return list(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + + +@pytest.mark.parametrize("dtype", [pa.uint32(), pa.uint64()]) +@pytest.mark.parametrize("selection", [[], [0], [1, 0, 1, 0]]) +def test_columnar_stable_deduplication_across_chunks(graph_task, monkeypatch, dtype, selection): + source = graph_task._assembler._source + original = source.columnar_graph_batch + raw_graphs = [] + + def capture(sample, spec, runtime, **kwargs): + graph = runtime["graphs"]["main"] + parts = [] + for edge in graph["edge_index_dict"]: + # Reorder and repeat valid pairs within and across native chunks. + assert len(edge["src"]) >= 2 + part = { + **edge, + "src": edge["src"].take(pa.array(selection, type=pa.int64())).cast(dtype), + "dst": edge["dst"].take(pa.array(selection, type=pa.int64())).cast(dtype), + } + parts.extend([part, part] if len(selection) > 1 else [part]) + runtime = {**runtime, "graphs": {"main": {**graph, "edge_index_dict": parts}}} + raw_graphs.append(original(sample, spec, runtime)) + return original(sample, spec, runtime, **kwargs) + + monkeypatch.setattr(source, "columnar_graph_batch", capture) + batch = next(graph_task.batches(batch_size=5, output_format="columnar", deduplicate_edges=True)) + for name, edge in batch.inputs["graph"].edges.items(): + raw = raw_graphs[0].edges[name] + pairs = _pairs(raw) + assert _pairs(edge) == list(dict.fromkeys(pairs)) + assert edge.index.schema == raw.index.schema + assert edge.index["source"].type == dtype + if len(selection) > 1: + assert len(pairs) > len(_pairs(edge)) + assert pairs[0] != pairs[1] + + +def test_columnar_deduplication_keeps_repeated_entities_separate(graph_task): + raw = next(graph_task.batches(batch_size=5, output_format="columnar")).inputs["graph"] + graph = next( + graph_task.batches(batch_size=5, output_format="columnar", deduplicate_edges=True) + ).inputs["graph"] + edge = graph.edges["activity"] + assert _pairs(edge) == list(dict.fromkeys(_pairs(raw.edges["activity"]))) + nodes = graph.nodes + by_identity = {} + for left, right in _pairs(edge): + owner = nodes[edge.source_type].owners[left].as_py() + assert owner == nodes[edge.destination_type].owners[right].as_py() + identity = ( + nodes[edge.source_type].ids[left].as_py(), + nodes[edge.destination_type].ids[right].as_py(), + ) + by_identity.setdefault(identity, set()).add(owner) + assert any(len(owners) > 1 for owners in by_identity.values()) + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +def test_deduplication_option_requires_columnar(graph_task, output_format): + with pytest.raises(ValueError, match="deduplicate_edges requires columnar"): + graph_task.batches(output_format=output_format, deduplicate_edges=True) + + +@pytest.mark.parametrize("value", [1, None, "false"]) +def test_deduplication_option_requires_bool(graph_task, value): + with pytest.raises(ValueError, match="deduplicate_edges must be a bool"): + graph_task.batches(output_format="columnar", deduplicate_edges=value) From 74aeaf86be398155b1115e0c49f62cefc41b1c92 Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:34:18 -0800 Subject: [PATCH 6/6] Fix executable columnar graph documentation example --- .../kumo-pytorch/docs/05-training-batches.md | 65 +++++++++++++++---- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/packages/experimental/kumo-pytorch/docs/05-training-batches.md b/packages/experimental/kumo-pytorch/docs/05-training-batches.md index 7348ffad7..ea4b8f384 100644 --- a/packages/experimental/kumo-pytorch/docs/05-training-batches.md +++ b/packages/experimental/kumo-pytorch/docs/05-training-batches.md @@ -387,18 +387,61 @@ For consumers that build their own model representation, a materialized DiskGraph task can expose sampled graph columns without first encoding them into tensor/mask dictionaries: +Reuse the schema, Parquet dataset and anchor table above with a graph-only +contract. Select the training split because the anchor table contains both +training and validation rows. + + ```python -# graph_task has one Graph input named "graph", explicit Raw node fields, -# and anchors materialized from an AnchorTable. -for batch in graph_task.batches(batch_size=1024, output_format="columnar"): - graph = batch.inputs["graph"] # api.ColumnarGraph - roots = graph.nodes[graph.root_type] - root_ids = roots.ids.slice(0, batch.batch_size) - for table_name, nodes in graph.nodes.items(): - features = nodes.features # pyarrow.Table; NULLs stay in validity bitmaps - example_positions = nodes.owners # indexes into batch.metadata["sample_ids"] - for relation_name, edges in graph.edges.items(): - selected_pairs = edges.index # Arrow columns: source, destination +graph_contract = api.DataLoaderSpec( + schema=schema, + sample=api.SampleSpec( + entity=api.EntitySpec(table="users"), + neighborhoods={ + "history": api.NeighborhoodSpec( + layers=( + api.GraphLayer( + samples=(api.GraphSample("user_purchases", 4, strategy="recent"),) + ), + ) + ) + }, + ), + batch=api.BatchSpec( + inputs={ + "graph": api.Input( + source=api.NeighborhoodSource("history"), + representation=api.Graph( + node_fields={ + "users": api.Fields((api.Encode("segment", api.Raw()),)), + "purchases": api.Fields((api.Encode("amount", api.Raw()),)), + } + ), + layout=None, + ) + }, + batching=api.Batching(max_samples=1024), + ), +) +with ( + api.prepare(graph_contract, parquet, engine) as graph_plan, + graph_plan.materialize(anchors=anchor_table, fit_split="train") as graph_task, +): + for batch in graph_task.batches(split="train", output_format="columnar"): + graph = batch.inputs["graph"] # api.ColumnarGraph + roots = graph.nodes[graph.root_type] + root_ids = roots.ids.slice(0, batch.batch_size) + for table_name, nodes in graph.nodes.items(): + features = nodes.features # pyarrow.Table; NULLs stay in validity bitmaps + example_positions = nodes.owners # indexes into batch.metadata["sample_ids"] + for relation_name, edges in graph.edges.items(): + selected_pairs = edges.index # Arrow columns: source, destination + print("training roots:", root_ids.to_pylist()) +``` + + +```text +training roots: [1, 1, 2, 2, 3, 3, 4, 4, 5, 5] ``` Node rows are occurrences within an example: repeated anchors and different