diff --git a/packages/experimental/kumo-pytorch/docs/05-training-batches.md b/packages/experimental/kumo-pytorch/docs/05-training-batches.md index d526f13ce..ea4b8f384 100644 --- a/packages/experimental/kumo-pytorch/docs/05-training-batches.md +++ b/packages/experimental/kumo-pytorch/docs/05-training-batches.md @@ -380,3 +380,92 @@ engine](04-engines-and-data.md#the-diskgraph-engine) for how the index is built and reused. Next: [inference and serving](06-inference-and-serving.md). + +## Arrow graphs for foundation-model payloads + +For consumers that build their own model representation, a materialized +DiskGraph task can expose sampled graph columns without first encoding them +into tensor/mask dictionaries: + +Reuse the schema, Parquet dataset and anchor table above with a graph-only +contract. Select the training split because the anchor table contains both +training and validation rows. + + +```python +graph_contract = api.DataLoaderSpec( + schema=schema, + sample=api.SampleSpec( + entity=api.EntitySpec(table="users"), + neighborhoods={ + "history": api.NeighborhoodSpec( + layers=( + api.GraphLayer( + samples=(api.GraphSample("user_purchases", 4, strategy="recent"),) + ), + ) + ) + }, + ), + batch=api.BatchSpec( + inputs={ + "graph": api.Input( + source=api.NeighborhoodSource("history"), + representation=api.Graph( + node_fields={ + "users": api.Fields((api.Encode("segment", api.Raw()),)), + "purchases": api.Fields((api.Encode("amount", api.Raw()),)), + } + ), + layout=None, + ) + }, + batching=api.Batching(max_samples=1024), + ), +) +with ( + api.prepare(graph_contract, parquet, engine) as graph_plan, + graph_plan.materialize(anchors=anchor_table, fit_split="train") as graph_task, +): + for batch in graph_task.batches(split="train", output_format="columnar"): + graph = batch.inputs["graph"] # api.ColumnarGraph + roots = graph.nodes[graph.root_type] + root_ids = roots.ids.slice(0, batch.batch_size) + for table_name, nodes in graph.nodes.items(): + features = nodes.features # pyarrow.Table; NULLs stay in validity bitmaps + example_positions = nodes.owners # indexes into batch.metadata["sample_ids"] + for relation_name, edges in graph.edges.items(): + selected_pairs = edges.index # Arrow columns: source, destination + print("training roots:", root_ids.to_pylist()) +``` + + +```text +training roots: [1, 1, 2, 2, 3, 3, 4, 4, 5, 5] +``` + +Node rows are occurrences within an example: repeated anchors and different +cutoffs do not share graph occurrences. The root table starts with one root +per batch example, in batch order. Edge endpoints index the node tables in +**declared left-to-right orientation**, including reverse traversal and +self-relations. They contain only sampled links. The same selected pair can +appear in multiple chunks. Consumers can normalize these pairs while building +their payload, or pass `deduplicate_edges=True` to `batches` for unique pairs +per relation in first-occurrence order. This opt-in step materializes edge +arrays and adds deduplication work; the default `False` retains native chunks. +Deduplication preserves separate examples, including repeated entity IDs. +Chunks do not identify traversal direction or discovery radius. + +Feature columns contain exactly the requested Raw fields, in declaration order +and declared types. Primary-key identity is available separately in `ids`. +Native buffers are shared where compatible; restoring declared types may +allocate. All buffers remain valid after iteration and task closure, and +`Batch.to(...)` keeps them as CPU Arrow values. + +This format currently supports one graph-only input, explicit Raw encodings, +fixed sample counts, and a selection contained in one split. It refuses +request-backed tasks, non-DiskGraph engines, other sample values, targets, +entity filters, and availability checks before iteration. Select a single +split explicitly when anchors span several splits. Shuffle, rank sharding, +padding, drop-last, and iterator resume retain their existing behavior. +Existing sampling limits, including timeless destinations, still apply. diff --git a/packages/experimental/kumo-pytorch/docs/api/README.md b/packages/experimental/kumo-pytorch/docs/api/README.md index a2ec41ebb..9b54cb972 100644 --- a/packages/experimental/kumo-pytorch/docs/api/README.md +++ b/packages/experimental/kumo-pytorch/docs/api/README.md @@ -71,6 +71,9 @@ The [guide](../README.md) explains the concepts in order. The behavior spec, wit | [`Fields`](batches.md#fields) | `kumo.pytorch.api` | Select ordered fields with explicit encoders, grouping, and field IDs. | | [`Encode`](batches.md#encode) | `kumo.pytorch.api` | Encode one sampled field, optionally translating IDs into a final space. | | [`Graph`](batches.md#graph) | `kumo.pytorch.api` | Node attributes use the same explicit field encodings as ordinary columns. | +| [`ColumnarGraph`](batches.md#columnargraph) | `kumo.pytorch.api` | CPU Arrow graph buffers emitted by `Task.batches(output_format="columnar")`. | +| [`ColumnarNodes`](batches.md#columnarnodes) | `kumo.pytorch.api` | Aligned columns for node occurrences, isolated per example. | +| [`ColumnarEdges`](batches.md#columnaredges) | `kumo.pytorch.api` | Selected edges in declared left-to-right relation orientation. | | [`Columns`](batches.md#columns) | `kumo.pytorch.api` | Keep selected encoded fields as separate typed columns. | | [`Tokens`](batches.md#tokens) | `kumo.pytorch.api` | Concatenate one row's encoded IDs in the declared field order. | | [`Composite`](batches.md#composite) | `kumo.pytorch.api` | One category ID for the ordered tuple of scalar component results. | diff --git a/packages/experimental/kumo-pytorch/docs/api/batches.md b/packages/experimental/kumo-pytorch/docs/api/batches.md index 27f24fe02..ce0e227d8 100644 --- a/packages/experimental/kumo-pytorch/docs/api/batches.md +++ b/packages/experimental/kumo-pytorch/docs/api/batches.md @@ -267,6 +267,71 @@ Node attributes use the same explicit field encodings as ordinary columns. - `node_fields`: table name to the `Fields` (scalar column encodings of physical columns) emitted for its nodes. Field-name IDs are not supported on graph nodes. +### `ColumnarGraph` + +*class* · `kumo.pytorch.api.ColumnarGraph` + +```python +class ColumnarGraph( + root_type: str, + nodes: Mapping[str, ColumnarNodes], + edges: Mapping[str, ColumnarEdges], +) +``` + +CPU Arrow graph buffers emitted by `Task.batches(output_format="columnar")`. + +`nodes` uses table names and `edges` uses declared relation names. +The first `Batch.batch_size` rows of `nodes[root_type]` are the roots, +in batch order, including repeated entities at distinct prediction times. +Isolated roots and empty declared node/edge types are retained. + +Buffers remain valid after advancing or closing the iterator/task. Treat +them as read-only. Compatible native Arrow buffers are shared; declared +dtype restoration may allocate. `Batch.to` leaves these CPU buffers in +Arrow form. This output avoids tensor encoding and is intended for callers +that construct their own model representation, such as foundation models. + +### `ColumnarNodes` + +*class* · `kumo.pytorch.api.ColumnarNodes` + +```python +class ColumnarNodes(ids: Array | ChunkedArray, owners: Array, features: Table) +``` + +Aligned columns for node occurrences, isolated per example. + +`ids` contains primary-key values (index-local row IDs when no primary +key is declared). `owners` is a non-null uint32 or uint64 Arrow array of positions +into `Batch.metadata["sample_ids"]`. The same entity may occur under +several owners. `features` is an Arrow table containing exactly the +declared Raw fields, in declaration order and declared dtypes. Arrow +validity bitmaps retain NULLs; no separate masks or filled values are used. +With no declared fields, the table has zero columns; use `len(ids)` for +the node count. All three outputs otherwise have the same row order. + +### `ColumnarEdges` + +*class* · `kumo.pytorch.api.ColumnarEdges` + +```python +class ColumnarEdges(source_type: str, destination_type: str, index: Table) +``` + +Selected edges in declared left-to-right relation orientation. + +`index` is an Arrow table with non-null uint32 or uint64 `source` and +`destination` columns, indexing rows in the corresponding node tables. +Endpoints always belong to the same example. Only edges selected by the +sampling layers are included, never all links among returned nodes. +Reverse traversals are reoriented, including self-relations. Chunks may +repeat a pair selected in both directions. Consumers must normalize these +pairs or request `Task.batches(..., deduplicate_edges=True)` to retain +the first occurrence per relation, at the cost of edge-array allocation. +Deduplication uses node positions, preserving separate examples even when +entity IDs repeat. Chunk order is not traversal provenance. + ### `Columns` *class* · `kumo.pytorch.api.Columns` @@ -482,8 +547,10 @@ never mixed. Separate columns sit under their field names with `values` and `valid`; token outputs use `value_ids`, `position_ids`, validity and source-event indices, with optional `field_ids` and `field_valid`. Raw types without a tensor representation stay Arrow arrays in inspectable -output. Metadata carries `sample_ids`, `entity_ids`, `as_of_time`, -`splits`, `is_padding`, request metadata, fingerprints and candidate +output. Columnar output holds `ColumnarGraph` objects with Arrow feature +tables and selected edge indexes. Metadata carries `sample_ids`, +`entity_ids`, `as_of_time`, `splits`, `is_padding`, request metadata, +fingerprints and candidate identities; it is not a route for undeclared features. #### `Batch.to` diff --git a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md index a66753428..85aefeb1d 100644 --- a/packages/experimental/kumo-pytorch/docs/api/lifecycle.md +++ b/packages/experimental/kumo-pytorch/docs/api/lifecycle.md @@ -388,7 +388,8 @@ Task.batches( world_size: int = 1, drop_last: bool | None = None, chunk_batches: int = 8, - output_format: Literal['inspectable', 'model'] = 'inspectable', + output_format: Literal['inspectable', 'model', 'columnar'] = 'inspectable', + deduplicate_edges: bool = False, ) -> TaskBatches ``` @@ -413,6 +414,17 @@ and `world_size`; ranks are disjoint before padding. A token budget representation before fetching a batch; `"inspectable"` keeps them as Arrow arrays. Both emit the same `Batch` structure. +`"columnar"` emits `ColumnarGraph` inputs with Arrow feature tables +and selected edge indexes, without tensor encoding. It requires +materialized DiskGraph anchors, one graph input with explicit Raw +fields, fixed sample counts, and a selection contained in one split. +Other sample values, targets, entity filters and availability checks +are refused before iteration. Shuffle, sharding, padding, drop-last +and resume retain their normal semantics. By default, selected edge pairs +may repeat across native chunks. `deduplicate_edges=True` is available +only for columnar output: it allocates edge arrays to retain the first +occurrence of each pair per relation, preserving example isolation. + #### `Task.preview` ```python diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py index 68083651e..36fd2540a 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/assembly.py @@ -2186,7 +2186,17 @@ def _validate_model_contract(self) -> None: "inspectable output or an explicit encoder" ) - def _collate(self, values: Any, *, epoch: int, seed: int, model=False, requests=()) -> Batch: + def _collate( + self, + values: Any, + *, + epoch: int, + seed: int, + model=False, + requests=(), + columnar=False, + deduplicate_edges=False, + ) -> Batch: declarations = { name: value.declarations for name, value in self._prepared_inputs.items() @@ -2206,6 +2216,14 @@ def _collate(self, values: Any, *, epoch: int, seed: int, model=False, requests= source=prepared.spec.source.name, node_fields=prepared.spec.representation.node_fields, ) + if columnar: + result[name] = self._source.columnar_graph_batch( + self.sample, + graph, + values.runtime_batch, + deduplicate_edges=deduplicate_edges, + ) + continue if model and getattr(values, "runtime_batch", None) is not None: result[name] = self._source.sample_graph_batch( self.sample, graph, prepared.state, values.runtime_batch diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py index 1aefa2450..916e8d211 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/__init__.py @@ -4,6 +4,7 @@ """Batch declarations: sources, explicit encodings, arrangements, and layouts.""" +from kumo.pytorch.contract.batch.columnar import ColumnarEdges, ColumnarGraph, ColumnarNodes from kumo.pytorch.contract.batch.encodings import ( Buckets, Raw, @@ -55,6 +56,9 @@ "Buckets", "CandidateSource", "CandidateTarget", + "ColumnarEdges", + "ColumnarGraph", + "ColumnarNodes", "Columns", "Composite", "Encode", diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py new file mode 100644 index 000000000..c97be0a3b --- /dev/null +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/columnar.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Arrow graph outputs for consumers that assemble their own model payloads.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + +import pyarrow as pa + + +@dataclass(frozen=True) +class ColumnarNodes: + """Aligned columns for node occurrences, isolated per example. + + ``ids`` contains primary-key values (index-local row IDs when no primary + key is declared). ``owners`` is a non-null uint32 or uint64 Arrow array of positions + into ``Batch.metadata["sample_ids"]``. The same entity may occur under + several owners. ``features`` is an Arrow table containing exactly the + declared Raw fields, in declaration order and declared dtypes. Arrow + validity bitmaps retain NULLs; no separate masks or filled values are used. + With no declared fields, the table has zero columns; use ``len(ids)`` for + the node count. All three outputs otherwise have the same row order. + """ + + ids: pa.Array | pa.ChunkedArray + owners: pa.Array + features: pa.Table + + +@dataclass(frozen=True) +class ColumnarEdges: + """Selected edges in declared left-to-right relation orientation. + + ``index`` is an Arrow table with non-null uint32 or uint64 ``source`` and + ``destination`` columns, indexing rows in the corresponding node tables. + Endpoints always belong to the same example. Only edges selected by the + sampling layers are included, never all links among returned nodes. + Reverse traversals are reoriented, including self-relations. Chunks may + repeat a pair selected in both directions. Consumers must normalize these + pairs or request ``Task.batches(..., deduplicate_edges=True)`` to retain + the first occurrence per relation, at the cost of edge-array allocation. + Deduplication uses node positions, preserving separate examples even when + entity IDs repeat. Chunk order is not traversal provenance. + """ + + source_type: str + destination_type: str + index: pa.Table + + +@dataclass(frozen=True) +class ColumnarGraph: + """CPU Arrow graph buffers emitted by ``Task.batches(output_format="columnar")``. + + ``nodes`` uses table names and ``edges`` uses declared relation names. + The first ``Batch.batch_size`` rows of ``nodes[root_type]`` are the roots, + in batch order, including repeated entities at distinct prediction times. + Isolated roots and empty declared node/edge types are retained. + + Buffers remain valid after advancing or closing the iterator/task. Treat + them as read-only. Compatible native Arrow buffers are shared; declared + dtype restoration may allocate. ``Batch.to`` leaves these CPU buffers in + Arrow form. This output avoids tensor encoding and is intended for callers + that construct their own model representation, such as foundation models. + """ + + root_type: str + nodes: Mapping[str, ColumnarNodes] + edges: Mapping[str, ColumnarEdges] diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py index dac1b125a..18ffcfce0 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/contract/batch/spec.py @@ -346,8 +346,10 @@ class Batch: and ``valid``; token outputs use ``value_ids``, ``position_ids``, validity and source-event indices, with optional ``field_ids`` and ``field_valid``. Raw types without a tensor representation stay Arrow arrays in inspectable - output. Metadata carries ``sample_ids``, ``entity_ids``, ``as_of_time``, - ``splits``, ``is_padding``, request metadata, fingerprints and candidate + output. Columnar output holds `ColumnarGraph` objects with Arrow feature + tables and selected edge indexes. Metadata carries ``sample_ids``, + ``entity_ids``, ``as_of_time``, ``splits``, ``is_padding``, request metadata, + fingerprints and candidate identities; it is not a route for undeclared features. """ diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py index b7ab39eaf..950c73cf6 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py @@ -1287,6 +1287,92 @@ def sample_graph_batch(self, sample, spec, state, runtime): finally: self._runtime_batch = previous + def columnar_graph_batch(self, sample, spec, runtime, *, deduplicate_edges=False): + """Expose sampled Arrow buffers without per-node Python or tensor assembly.""" + from kumo.pytorch.contract.batch import ColumnarEdges, ColumnarGraph, ColumnarNodes + + pa = self._pa + graph = runtime["graphs"]["main"] + neighborhood = sample.neighborhoods[spec.source] + _, physical_relations = self._sampling_plan(neighborhood) + relations = { + traversal.relation for layer in neighborhood.layers for traversal in layer.samples + } + tables = {sample.entity.table, *spec.node_fields} + for name in relations: + relation = self._relations[name] + tables.update((relation.left_table, relation.right_table)) + nodes = {} + for table in sorted(tables): + ids = graph["node_ids"].get(table, pa.array([], type=pa.uint64())) + owners = graph["batch_dict"].get(table, pa.array([], type=pa.uint64())) + fields = spec.node_fields.get(table) + names = () if fields is None else tuple(use.field for use in fields.fields) + definition = self._tables[table] + requested = tuple( + dict.fromkeys( + (*(() if definition.primary_key is None else (definition.primary_key,)), *names) + ) + ) + raw = runtime["raw_features"].get(table) + if len(owners) != len(ids): + raise RuntimeError(f"native columnar node/owner alignment mismatch: {table}") + if len(ids) and requested: + if raw is None: + raise RuntimeError(f"missing native columnar features for {table!r}") + data = pa.Table.from_batches(raw["record_batches"]) + if len(data) != len(ids) or len(owners) != len(ids): + raise RuntimeError(f"native columnar node/feature alignment mismatch: {table}") + columns = { + name: self._typed_array(data[name], definition.columns[name]) + for name in requested + } + else: + columns = { + name: pa.array([], type=definition.columns[name].to_arrow()) + for name in requested + } + nodes[table] = ColumnarNodes( + ids=columns[definition.primary_key] if definition.primary_key else ids, + owners=owners, + features=pa.table({name: columns[name] for name in names}), + ) + parts = defaultdict(list) + for edge in graph["edge_index_dict"]: + physical = edge["edge_type"]["rel"] + if physical not in physical_relations: + continue + name = physical_relations[physical] + left, right = edge["src"], edge["dst"] + if physical != self._relation_mapping[name]["forward"]: + left, right = right, left + parts[name].append(pa.record_batch([left, right], names=["source", "destination"])) + empty = pa.schema([("source", pa.uint64()), ("destination", pa.uint64())]) + edges = {} + for name in sorted(relations): + index = ( + pa.Table.from_batches(parts[name]) + if parts[name] + else pa.Table.from_batches([], schema=empty) + ) + if deduplicate_edges and len(index) > 1: + # Opt-in materialization; the default retains native edge chunks. + pairs = _first_occurrences( + np.stack([column.to_numpy() for column in index.columns]) + ) + index = pa.table( + { + field.name: pa.array(values, type=field.type) + for field, values in zip(index.schema, pairs, strict=True) + } + ) + edges[name] = ColumnarEdges( + self._relations[name].left_table, + self._relations[name].right_table, + index, + ) + return ColumnarGraph(sample.entity.table, nodes, edges) + def build_model_batch(self, spec, runtime, anchors, compiled): """Adapt iterator buffers; history work has finished before Python resumes.""" if self._sampler is None: diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py index 1eedb5d92..56df0aa2c 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/native_anchors.py @@ -776,6 +776,7 @@ def _typed_batches( world_size, drop_last, output_format, + deduplicate_edges=False, ): """One native iterator call per batch; no request objects or re-batching.""" from dataclasses import replace @@ -901,6 +902,18 @@ def runtime_batches(): for meta, flag in zip(anchors.request_metadata, flags, strict=True) ), ) + if output_format == "columnar": + yield _surface_padding( + assembler._collate( + replace(anchors, runtime_batch=runtime), + epoch=epoch, + seed=seed, + model=True, + columnar=True, + deduplicate_edges=deduplicate_edges, + ) + ) + continue with runtime_features(source, runtime): values = source.build_model_batch(sample, runtime, anchors, compiled) values = replace( @@ -1043,6 +1056,27 @@ def close(self) -> None: self._event_costs = {} self._temporary.cleanup() + def validate_columnar(self, assembler, selection): + """Refuse unsupported contracts before constructing a native iterator.""" + from kumo.pytorch.contract.batch import Raw + + policy = assembler.spec.batching + if ( + not self._can_batch_inspectable_graph(assembler) + or policy.max_tokens is not None + or policy.bucket_by_length + or len(self._selection(selection)) > 1 + or assembler._source is not self._source + ): + raise ValueError( + "columnar output requires one graph-only input, fixed-size batching, " + "one selected split, and no targets, entity filters or availability" + ) + for prepared in assembler._prepared_inputs.values(): + for fields in prepared.spec.representation.node_fields.values(): + if any(not isinstance(use.encoding, Raw) for use in fields.fields): + raise ValueError("columnar graph fields require explicit Raw encodings") + def _can_batch_inspectable_graph(self, assembler): """Use typed buffers only when no other inspectable contract is needed.""" from kumo.pytorch.contract.batch import Graph @@ -1084,6 +1118,7 @@ def batches( world_size: int, drop_last: bool | None, output_format: str, + deduplicate_edges: bool = False, ): """Read native anchor batches and retain the existing fitted assembler. @@ -1097,6 +1132,8 @@ def batches( from kumo.pytorch.contract.sample import Field from kumo.pytorch.task import _mark_padding, _surface_padding + if output_format == "columnar": + self.validate_columnar(assembler, selection) source = self._source policy = assembler.spec.batching fixed = policy.max_tokens is None and not policy.bucket_by_length @@ -1176,7 +1213,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): and self._can_batch_inspectable_graph(assembler) ) if ( - (output_format == "model" or inspectable_graph) + (output_format in {"model", "columnar"} or inspectable_graph) and (fixed or output_format == "model" and world_size == 1 and not neighborhoods) and assembler._source is self._source ): @@ -1199,6 +1236,7 @@ def assemble(requests: Sequence[SampleRequest], runtime: Any, final: bool): world_size=world_size, drop_last=policy.drop_last if drop_last is None else drop_last, output_format=output_format, + deduplicate_edges=deduplicate_edges, ) return diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py index a4b27c564..b9d4a5689 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/task.py @@ -209,7 +209,8 @@ def batches( world_size: int = 1, drop_last: bool | None = None, chunk_batches: int = 8, - output_format: Literal["inspectable", "model"] = "inspectable", + output_format: Literal["inspectable", "model", "columnar"] = "inspectable", + deduplicate_edges: bool = False, ) -> TaskBatches: """Emit batches over this task's anchors, or over requests built in code. @@ -231,6 +232,17 @@ def batches( ``output_format="model"`` refuses raw types without a tensor representation before fetching a batch; ``"inspectable"`` keeps them as Arrow arrays. Both emit the same `Batch` structure. + + ``"columnar"`` emits `ColumnarGraph` inputs with Arrow feature tables + and selected edge indexes, without tensor encoding. It requires + materialized DiskGraph anchors, one graph input with explicit Raw + fields, fixed sample counts, and a selection contained in one split. + Other sample values, targets, entity filters and availability checks + are refused before iteration. Shuffle, sharding, padding, drop-last + and resume retain their normal semantics. By default, selected edge pairs + may repeat across native chunks. ``deduplicate_edges=True`` is available + only for columnar output: it allocates edge arrays to retain the first + occurrence of each pair per relation, preserving example isolation. """ if requests is not None and split is not None: @@ -264,8 +276,18 @@ def batches( "chunk_batches": chunk_batches, "output_format": output_format, } - if output_format not in {"inspectable", "model"}: - raise ValueError("output_format must be 'inspectable' or 'model'") + if output_format not in {"inspectable", "model", "columnar"}: + raise ValueError("output_format must be 'inspectable', 'model', or 'columnar'") + if not isinstance(deduplicate_edges, bool): + raise ValueError("deduplicate_edges must be a bool") + if deduplicate_edges and output_format != "columnar": + raise ValueError("deduplicate_edges requires columnar output") + if output_format == "columnar": + config["deduplicate_edges"] = deduplicate_edges + validate = getattr(self._anchors, "validate_columnar", None) + if requests is not None or not callable(validate): + raise ValueError("columnar output requires materialized DiskGraph anchors") + validate(self._assembler, selection) if requests is None: self._anchors.count(selection) # an unknown split is refused now, not mid-iteration max_samples = ( @@ -289,6 +311,7 @@ def produce() -> Iterator[Batch]: world_size=world_size, drop_last=drop_last, output_format=output_format, + deduplicate_edges=deduplicate_edges, ) else: @@ -322,7 +345,8 @@ def _anchor_batches( rank: int, world_size: int, drop_last: bool | None, - output_format: Literal["inspectable", "model"] = "inspectable", + output_format: Literal["inspectable", "model", "columnar"] = "inspectable", + deduplicate_edges: bool = False, ) -> Iterator[Batch]: anchors = self._anchors view = getattr(anchors, "iteration_anchors", None) @@ -342,6 +366,7 @@ def _anchor_batches( world_size=world_size, drop_last=drop_last, output_format=output_format, + **({"deduplicate_edges": deduplicate_edges} if output_format == "columnar" else {}), ) return positions = padded_positions( diff --git a/packages/experimental/kumo-pytorch/tests/api_reference.py b/packages/experimental/kumo-pytorch/tests/api_reference.py index a4c674667..5616d7c3b 100644 --- a/packages/experimental/kumo-pytorch/tests/api_reference.py +++ b/packages/experimental/kumo-pytorch/tests/api_reference.py @@ -107,6 +107,9 @@ class Page: "Fields", "Encode", "Graph", + "ColumnarGraph", + "ColumnarNodes", + "ColumnarEdges", "Columns", "Tokens", "Composite", diff --git a/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py new file mode 100644 index 000000000..e34fd3a2e --- /dev/null +++ b/packages/experimental/kumo-pytorch/tests/test_columnar_graph_batches.py @@ -0,0 +1,373 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Columnar graphs preserve selected occurrences and bypass tensor assembly.""" + +import pyarrow as pa +import pyarrow.compute as pc +import pytest +import torch + +from kumo.pytorch import api +from kumo.pytorch.engines.native_anchors import NativeAnchors +from tests.test_inspectable_graph_batches import _equal +from tests.test_inspectable_graph_batches import graph_task as graph_task + + +def _compare(actual, expected): + _equal(actual.metadata, expected.metadata) + _equal(actual.targets, expected.targets) + graph = actual.inputs["graph"] + old = expected.inputs["graph"] + assert isinstance(graph, api.ColumnarGraph) + assert graph.root_type == "users" + assert graph.nodes.keys() == old["nodes"].keys() + assert graph.edges.keys() == old["edges"].keys() + for name, node in graph.nodes.items(): + previous = old["nodes"][name] + ids = previous["ids"] + ids = ids.tolist() if isinstance(ids, torch.Tensor) else ids.to_pylist() + assert node.ids.to_pylist() == ids + assert node.owners.type in (pa.uint32(), pa.uint64()) + assert node.owners.to_pylist() == previous["batch"].tolist() + assert node.features.column_names == list(previous["features"]) + for field in node.features.column_names: + values = previous["features"][field]["values"] + if isinstance(values, torch.Tensor): + values = pa.array(values.numpy()) + valid = pa.array(previous["features"][field]["valid"].numpy()) + restored = pc.if_else(valid, values, None) + assert node.features[field].combine_chunks().equals(restored) + for name, edge in graph.edges.items(): + previous = old["edges"][name] + assert edge.source_type == previous["source_type"] + assert edge.destination_type == previous["destination_type"] + pairs = set(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + assert pairs == set(map(tuple, previous["index"].numpy().T)) + for left, right in pairs: + assert ( + graph.nodes[edge.source_type].owners[left].as_py() + == graph.nodes[edge.destination_type].owners[right].as_py() + ) + + +@pytest.mark.parametrize("deduplicate_edges", [False, True]) +@pytest.mark.parametrize("world_size", [1, 3, 7]) +@pytest.mark.parametrize("shuffle", [False, True]) +@pytest.mark.parametrize("drop_last", [False, True]) +def test_columnar_parity_and_lifetime( + graph_task, monkeypatch, world_size, shuffle, drop_last, deduplicate_edges +): + task = graph_task + options = dict(world_size=world_size, shuffle=shuffle, drop_last=drop_last, seed=5) + expected = [list(task.batches(rank=rank, **options)) for rank in range(world_size)] + + def forbidden(*args, **kwargs): + raise AssertionError("columnar output rebuilt requests or encoded a public graph") + + with monkeypatch.context() as patch: + patch.setattr(NativeAnchors, "_requests", forbidden) + patch.setattr(task._assembler._source, "_graph_output", forbidden) + patch.setattr(task._assembler._source, "build_model_batch", forbidden) + actual = [ + list( + task.batches( + rank=rank, + output_format="columnar", + deduplicate_edges=deduplicate_edges, + **options, + ) + ) + for rank in range(world_size) + ] + # Comparing after all iterators finish also checks native buffer lifetime. + for batches, old in zip(actual, expected, strict=True): + assert len(batches) == len(old) + for batch, previous in zip(batches, old, strict=True): + _compare(batch, previous) + _compare(batch.to("cpu"), previous) + if deduplicate_edges: + for edge in batch.inputs["graph"].edges.values(): + pairs = _pairs(edge) + assert len(pairs) == len(set(pairs)) + + +def test_columnar_reuses_native_buffers(graph_task, monkeypatch): + source = graph_task._assembler._source + original = source.columnar_graph_batch + captured = [] + + def capture(sample, spec, runtime, **kwargs): + captured.append(runtime) + return original(sample, spec, runtime, **kwargs) + + def forbidden(*args, **kwargs): + raise AssertionError("default columnar output attempted deduplication") + + monkeypatch.setattr("kumo.pytorch.engines.diskgraph._first_occurrences", forbidden) + monkeypatch.setattr(source, "columnar_graph_batch", capture) + batches = list(graph_task.batches(output_format="columnar")) + for batch, runtime in zip(batches, captured, strict=True): + graph = batch.inputs["graph"] + native_edge_buffers = { + values.buffers()[1].address + for edge in runtime["graphs"]["main"]["edge_index_dict"] + for values in (edge["src"], edge["dst"]) + if len(values) + } + for edge in graph.edges.values(): + for column in edge.index.columns: + for chunk in column.chunks: + if len(chunk): + assert chunk.buffers()[1].address in native_edge_buffers + for name, node in graph.nodes.items(): + if not len(node.ids): + continue + assert ( + node.owners.buffers()[1].address + == runtime["graphs"]["main"]["batch_dict"][name].buffers()[1].address + ) + if name == "events": + raw = pa.Table.from_batches(runtime["raw_features"][name]["record_batches"]) + for field in ("note",): + assert ( + node.features[field].chunk(0).buffers()[1].address + == raw[field].chunk(0).buffers()[1].address + ) + + +@pytest.mark.parametrize("graph_task", ["mixed", "availability"], indirect=True) +def test_unsupported_columnar_contract_fails_before_iteration(graph_task, monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("unsupported contract opened native iterator") + + monkeypatch.setattr(NativeAnchors, "_iter", forbidden) + with pytest.raises(ValueError, match="columnar output requires"): + graph_task.batches(output_format="columnar") + + +def test_columnar_rejects_requests(graph_task): + with pytest.raises(ValueError, match="materialized DiskGraph anchors"): + graph_task.batches(requests=[], output_format="columnar") + + +@pytest.mark.parametrize("deduplicate_edges", [False, True]) +def test_columnar_resume_and_closed_task_buffers(graph_task, deduplicate_edges): + task = graph_task + first = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=deduplicate_edges + ) + next(first) + state = first.state_dict() + changed = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=not deduplicate_edges + ) + with pytest.raises(ValueError, match="different batch configuration"): + changed.load_state_dict(state) + expected = list(first) + resumed = task.batches( + output_format="columnar", shuffle=True, seed=9, deduplicate_edges=deduplicate_edges + ) + resumed.load_state_dict(state) + actual = list(resumed) + assert [b.metadata["sample_ids"] for b in actual] == [ + b.metadata["sample_ids"] for b in expected + ] + for batch, previous in zip(actual, expected, strict=True): + for name, edge in batch.inputs["graph"].edges.items(): + assert edge.index.equals(previous.inputs["graph"].edges[name].index) + snapshots = [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] + task.close() + assert [b.inputs["graph"].nodes["events"].features.to_pylist() for b in actual] == snapshots + + +@pytest.mark.parametrize("deduplicate_edges", [False, True]) +@pytest.mark.parametrize("pk_left", [True, False]) +@pytest.mark.parametrize("directions", [("forward",), ("reverse",), ("forward", "reverse")]) +def test_columnar_self_relations_follow_declared_endpoints( + tmp_path, pk_left, directions, deduplicate_edges +): + import pyarrow.parquet as pq + + from conformance.assertions import bound_tables + + rows = [ + {"id": 1, "parent_id": None, "at": 1}, + {"id": 2, "parent_id": 1, "at": 10}, + {"id": 3, "parent_id": 2, "at": 20}, + ] + left_key, right_key = ("id", "parent_id") if pk_left else ("parent_id", "id") + contract = api.DataLoaderSpec( + api.DataSchema( + ( + api.TableSpec( + "accounts", + "id", + { + "id": api.DType("int64"), + "parent_id": api.DType("int64"), + "at": api.DType("timestamp", unit="s", timezone="UTC"), + }, + time_column="at", + ), + ), + (api.RelationSpec("parent", "accounts", left_key, "accounts", right_key),), + ), + api.SampleSpec( + api.EntitySpec("accounts"), + neighborhoods={ + "family": api.NeighborhoodSpec( + tuple( + api.GraphLayer( + tuple( + api.GraphSample("parent", 4, direction=d, strategy="recent") + for d in directions + ) + ) + for _ in range(2) + ) + ) + }, + ), + api.BatchSpec( + { + "graph": api.Input( + api.NeighborhoodSource("family"), + api.Graph({"accounts": api.Fields((api.Encode("parent_id", api.Raw()),))}), + None, + ) + } + ), + ) + anchors = tmp_path / "anchors.parquet" + pq.write_table( + pa.table({"id": [2, 2], "cutoff": pa.array([100, 5], type=pa.timestamp("s", tz="UTC"))}), + anchors, + ) + with ( + bound_tables(contract, {"accounts": rows}, "diskgraph") as (dataset, engine), + api.prepare(contract, dataset, engine) as plan, + plan.materialize(api.AnchorTable(api.Parquet(str(anchors)), "id", "cutoff")) as task, + ): + old = next(iter(task.batches(batch_size=2))) + raw = next(task.batches(batch_size=2, output_format="columnar")).inputs["graph"] + graph = next( + task.batches( + batch_size=2, output_format="columnar", deduplicate_edges=deduplicate_edges + ) + ).inputs["graph"] + if deduplicate_edges: + assert _pairs(graph.edges["parent"]) == list(dict.fromkeys(_pairs(raw.edges["parent"]))) + node = graph.nodes["accounts"] + edge = graph.edges["parent"] + assert node.ids.to_pylist()[:2] == [2, 2] + pairs = list(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + assert set(pairs) == set(map(tuple, old.inputs["graph"]["edges"]["parent"]["index"].numpy().T)) + by_id = {row["id"]: row for row in rows} + for a, b in pairs: + assert by_id[node.ids[a].as_py()][left_key] == by_id[node.ids[b].as_py()][right_key] + assert node.owners[a].as_py() == node.owners[b].as_py() + + +@pytest.mark.parametrize("unsupported", ["encoding", "tokens"]) +def test_columnar_refuses_non_raw_and_token_budget(graph_task, monkeypatch, unsupported): + from dataclasses import replace + + assembler = graph_task._assembler + if unsupported == "encoding": + prepared = assembler._prepared_inputs["graph"] + graph = prepared.spec.representation + fields = dict(graph.node_fields) + fields["events"] = api.Fields((api.Encode("note", api.Vocabulary()),)) + monkeypatch.setitem( + assembler._prepared_inputs, + "graph", + replace( + prepared, + spec=replace(prepared.spec, representation=replace(graph, node_fields=fields)), + ), + ) + else: + monkeypatch.setattr( + assembler, + "spec", + replace( + assembler.spec, + batching=api.Batching(max_samples=2, max_tokens=10, length_input="graph"), + ), + ) + with pytest.raises(ValueError, match="columnar"): + graph_task.batches(output_format="columnar") + + +def _pairs(edge): + return list(zip(edge.index["source"].to_pylist(), edge.index["destination"].to_pylist())) + + +@pytest.mark.parametrize("dtype", [pa.uint32(), pa.uint64()]) +@pytest.mark.parametrize("selection", [[], [0], [1, 0, 1, 0]]) +def test_columnar_stable_deduplication_across_chunks(graph_task, monkeypatch, dtype, selection): + source = graph_task._assembler._source + original = source.columnar_graph_batch + raw_graphs = [] + + def capture(sample, spec, runtime, **kwargs): + graph = runtime["graphs"]["main"] + parts = [] + for edge in graph["edge_index_dict"]: + # Reorder and repeat valid pairs within and across native chunks. + assert len(edge["src"]) >= 2 + part = { + **edge, + "src": edge["src"].take(pa.array(selection, type=pa.int64())).cast(dtype), + "dst": edge["dst"].take(pa.array(selection, type=pa.int64())).cast(dtype), + } + parts.extend([part, part] if len(selection) > 1 else [part]) + runtime = {**runtime, "graphs": {"main": {**graph, "edge_index_dict": parts}}} + raw_graphs.append(original(sample, spec, runtime)) + return original(sample, spec, runtime, **kwargs) + + monkeypatch.setattr(source, "columnar_graph_batch", capture) + batch = next(graph_task.batches(batch_size=5, output_format="columnar", deduplicate_edges=True)) + for name, edge in batch.inputs["graph"].edges.items(): + raw = raw_graphs[0].edges[name] + pairs = _pairs(raw) + assert _pairs(edge) == list(dict.fromkeys(pairs)) + assert edge.index.schema == raw.index.schema + assert edge.index["source"].type == dtype + if len(selection) > 1: + assert len(pairs) > len(_pairs(edge)) + assert pairs[0] != pairs[1] + + +def test_columnar_deduplication_keeps_repeated_entities_separate(graph_task): + raw = next(graph_task.batches(batch_size=5, output_format="columnar")).inputs["graph"] + graph = next( + graph_task.batches(batch_size=5, output_format="columnar", deduplicate_edges=True) + ).inputs["graph"] + edge = graph.edges["activity"] + assert _pairs(edge) == list(dict.fromkeys(_pairs(raw.edges["activity"]))) + nodes = graph.nodes + by_identity = {} + for left, right in _pairs(edge): + owner = nodes[edge.source_type].owners[left].as_py() + assert owner == nodes[edge.destination_type].owners[right].as_py() + identity = ( + nodes[edge.source_type].ids[left].as_py(), + nodes[edge.destination_type].ids[right].as_py(), + ) + by_identity.setdefault(identity, set()).add(owner) + assert any(len(owners) > 1 for owners in by_identity.values()) + + +@pytest.mark.parametrize("output_format", ["inspectable", "model"]) +def test_deduplication_option_requires_columnar(graph_task, output_format): + with pytest.raises(ValueError, match="deduplicate_edges requires columnar"): + graph_task.batches(output_format=output_format, deduplicate_edges=True) + + +@pytest.mark.parametrize("value", [1, None, "false"]) +def test_deduplication_option_requires_bool(graph_task, value): + with pytest.raises(ValueError, match="deduplicate_edges must be a bool"): + graph_task.batches(output_format="columnar", deduplicate_edges=value)