From d8768d888be974f83de26a0365b7e0cfd294efe0 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Fri, 13 Nov 2026 13:18:24 -0800 Subject: [PATCH] Preserve columnar external-loader APIs on the kumo-engine stack --- packages/diskgraph/diskgraph-pybind/README.md | 87 +++++ .../python/tests/bench_columnar_requests.py | 132 +++++++ .../python/tests/test_columnar_sampling.py | 231 ++++++++++++ .../diskgraph-pybind/src/columnar_input.rs | 107 ++++++ .../diskgraph/diskgraph-pybind/src/lib.rs | 329 +++++++++++------- 5 files changed, 761 insertions(+), 125 deletions(-) create mode 100644 packages/diskgraph/diskgraph-pybind/python/tests/bench_columnar_requests.py create mode 100644 packages/diskgraph/diskgraph-pybind/python/tests/test_columnar_sampling.py create mode 100644 packages/diskgraph/diskgraph-pybind/src/columnar_input.rs diff --git a/packages/diskgraph/diskgraph-pybind/README.md b/packages/diskgraph/diskgraph-pybind/README.md index 3189110e1..e01523a1d 100644 --- a/packages/diskgraph/diskgraph-pybind/README.md +++ b/packages/diskgraph/diskgraph-pybind/README.md @@ -146,6 +146,93 @@ The fluent materializer windows the selected destination frame and rejects temporal intermediate tables. This does not change PQL's first-hop window policy or add arbitrary multi-hop temporal support. +### Columnar requests for external loaders + +Callers that own an anchor table can keep its columns out of Python row objects: + +```python +import json +from collections import defaultdict + +import diskgraph +import pyarrow as pa + +engine = diskgraph.SamplerEngine("/path/to/index", num_threads=4) +# Bound each request upstream; this is one batch, not the entire anchor table. +keys = pa.array(["u1", "u2", "u1"]) +cutoffs = pa.array([1609545600, 1609632000, 1609718400], type=pa.int64()) + +# The caller supplies its normal SamplingPlan JSON. +events = engine.sample_from_keys( + {"users": keys}, plan_json, fused=True, + seed_timestamps={"users": cutoffs}, + seed_timestamp_unit=diskgraph.TimeUnit.Seconds, + swap_edge_direction=False, +) +batches = defaultdict(list) +for event in events: + if event["type"] != "stats": + kind = event["type"] + type_id = event["rel_id"] if kind == "edge" else event["type_id"] + batches[kind, type_id].append(event["batch"]) +tables = {key: pa.Table.from_batches(parts) for key, parts in batches.items()} +``` + +`sample_from_keys` resolves the supplied typed Arrow primary keys and invokes +the existing sampler. `fused=True` also fetches features in that same request. +It returns the same node, edge, feature and stats events as `sample`; it does +not change fanouts, randomness, temporal selection or edge direction. Repeated +keys retain separate anchor positions and aligned cutoffs. A missing key raises +`DiskgraphDataError` with its type and batch offset before sampling starts; +keys are never silently dropped. Invalid/null keys keep `lookup_node_ids`'s +existing errors. No second key map or per-anchor Python request is constructed. + +For reusable/materialized dense IDs, resolve once and use `sample` directly: + +```python +ids = engine.lookup_node_ids("users", keys, output_format="model") +# Missing keys are nulls in the uint64 array. Apply the caller's missing-key +# policy to all aligned columns before sampling; sample rejects null IDs. +events = engine.sample( + {"users": ids}, plan_json, fused=True, + seed_timestamps={"users": cutoffs}, + seed_timestamp_unit=diskgraph.TimeUnit.Seconds, +) +``` + +`lookup_node_ids` defaults to the existing Python list result; +`output_format="model"` returns nullable Arrow `uint64`. Both preserve input +order, repetitions and missing-key positions. IDs belong to the opened index; +persisted IDs need the caller's snapshot provenance checks. + +`sample`, `fetch_features`, `canonical_node_mask`, `query_temporal` and +`query_temporal_lengths` accept dense IDs as Arrow `uint64` or nonnegative +`int64` arrays as well as their existing Python sequences. Cutoffs accept Arrow +`int64` or the existing sequences. Arrays are imported through Arrow's C data +interface, including sliced arrays, without Python scalar iteration. Pass one +array per column; combine a `ChunkedArray` explicitly. Nulls and incompatible +physical types are rejected, including timestamp-typed arrays: cast cutoffs +explicitly to the API's declared integer time unit. For `sample` and +`sample_from_keys`, the existing default unit is **nanoseconds**; temporal +queries always take **seconds**. Supplied sampling cutoff columns must match +their seed type and length; omitted types retain the existing unbounded behavior. +The engine still copies these input buffers into owned native vectors for the +request lifetime. This is not an end-to-end zero-copy guarantee. + +Keep the returned Arrow batches through downstream gathering/encoding. In node +events, `id_a` is the owner seed position and `id_b` is the dense graph ID; +edge endpoints are local positions in the corresponding node tables, subject +to `swap_edge_direction`. Features retain the existing node-row alignment. +An FM payload builder can gather these columns with Arrow and convert only at +its wire boundary; a tensor consumer can use numeric Arrow/NumPy buffers. +Check nulls/dtypes and copy when mutable tensor storage is required. Python +event dictionaries remain per batch, not per node or feature value. + +These APIs support external loaders such as FM without requiring task-owned +anchor materialization. The kumo.pytorch native anchor/model-batch path remains +available separately. No task policy, model dependency, or FM wire schema is +added here. + ### Direct temporal rows `SamplerEngine.query_temporal(seed_nodes, seed_timestamps, plan_json, diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/bench_columnar_requests.py b/packages/diskgraph/diskgraph-pybind/python/tests/bench_columnar_requests.py new file mode 100644 index 000000000..c78b40915 --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/python/tests/bench_columnar_requests.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Matched request-boundary probe; not an FM payload or model benchmark. + +Run with an installed extension from this checkout: + python diskgraph-pybind/python/tests/bench_columnar_requests.py --output result.json + +Uses a fresh synthetic index, one native thread, warm caches, batch 512, and +alternating path order. Ingestion, lookup precomputation and exact Arrow-output +checks are outside timing. All paths consume the same fused Arrow event stream. +""" + +import argparse +from collections import defaultdict +import hashlib +import json +from pathlib import Path +import statistics +import tempfile +import time + +import diskgraph +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + + +def fixture(root): + users, per_user = 2048, 16 + pq.write_table(pa.table({"id": np.arange(users, dtype=np.int64)}), root / "users.parquet") + pq.write_table(pa.table({ + "id": np.arange(users * per_user, dtype=np.int64), + "owner": np.repeat(np.arange(users, dtype=np.int64), per_user), + "amount": np.arange(users * per_user, dtype=np.float64) / 4, + }), root / "events.parquet") + tables = [] + for name, columns in [("users", [("id", "int64")]), + ("events", [("id", "int64"), ("owner", "int64"), ("amount", "float64")])]: + table = {"name": name, "primary_key": "id", + "columns": [{"name": col, "ingested_dtype": dtype} for col, dtype in columns], + "source": {"type": "parquet", "paths": [str(root / f"{name}.parquet")]}} + if name == "events": + table["fkeys"] = [{"src_column": "owner", "dst_table": "users", "reverse": {}}] + tables.append(table) + diskgraph.ingest(json.dumps({"dataset_name": "columnar_probe", "num_shards": 1, + "tables": tables}), str(root / "index"), col_stats=False) + return diskgraph.SamplerEngine(str(root / "index"), random_seed=41, num_threads=1) + + +def consume(events, verify): + batches = defaultdict(list) + rows = 0 + for event in events: + if event["type"] == "stats": + continue + rows += event["batch"].num_rows + if verify: + batches[event["type"], event.get("rel_id", event.get("type_id"))].append(event["batch"]) + digest = hashlib.sha256() + for key, parts in sorted(batches.items()): + table = pa.Table.from_batches(parts).combine_chunks() + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema) as writer: + writer.write_table(table) + digest.update(str(key).encode()) + digest.update(sink.getvalue()) + return rows, digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--anchors", type=int, nargs="+", default=[32000, 400000]) + parser.add_argument("--repeats", type=int, default=5) + args = parser.parse_args() + results = [] + with tempfile.TemporaryDirectory(prefix="columnar-probe-") as tmp: + engine = fixture(Path(tmp)) + relation = next(r["name"] for r in json.loads(engine.get_type_mapping())["edge_relations"] + if r["src_type"] == "users") + plan = json.dumps({"layers": [{"fanouts": {relation: { + "fanout": 16, "mode": "deterministic_uniform", "absent_key_policy": "uniform_fallback", + }}, "default_fanout": None}]}) + paths = ["lookup_lists", "lookup_arrays", "lookup_fused", "materialized_lists", "materialized_arrays"] + for count in args.anchors: + keys = pa.array(np.arange(count, dtype=np.int64) % 2048) + ids = engine.lookup_node_ids("users", keys, output_format="model") + cutoffs = pa.array(np.full(count, 1700000000, dtype=np.int64)) + def run(path, verify=False): + result = [] + for start in range(0, count, 512): + chunk = keys.slice(start, 512) + times = cutoffs.slice(start, 512) + options = dict(fused=True, seed_timestamp_unit=diskgraph.TimeUnit.Seconds) + if path == "lookup_fused": + events = engine.sample_from_keys({"users": chunk}, plan, + seed_timestamps={"users": times}, **options) + else: + roots = (engine.lookup_node_ids("users", chunk, output_format=( + "model" if path == "lookup_arrays" else "inspectable")) + if path in {"lookup_lists", "lookup_arrays"} else ids.slice(start, 512)) + if path == "materialized_lists": + roots = roots.to_pylist() + if path not in {"materialized_arrays", "lookup_arrays"}: + times = times.to_pylist() + events = engine.sample({"users": roots}, plan, + seed_timestamps={"users": times}, **options) + result.append(consume(events, verify)) + return result + # Every batch on every path is checked, including per-anchor ownership. + expected = run(paths[0], verify=True) + for path in paths[1:]: + assert run(path, verify=True) == expected, path + for path in paths: + run(path) # untimed warmup + samples = {path: [] for path in paths} + for repeat in range(args.repeats): + for path in paths if repeat % 2 == 0 else paths[::-1]: + start = time.perf_counter() + run(path) + samples[path].append(time.perf_counter() - start) + results.append({"anchors": count, "batch_size": 512, "exact_output_parity": True, + "seconds": samples, + "medians": {p: statistics.median(v) for p, v in samples.items()}}) + args.output.write_text(json.dumps(results, indent=2) + "\n") + print(json.dumps(results[-1]), flush=True) + + +if __name__ == "__main__": + main() diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_columnar_sampling.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_columnar_sampling.py new file mode 100644 index 000000000..1b1ea1043 --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_columnar_sampling.py @@ -0,0 +1,231 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Columnar request boundaries preserve the existing graph/temporal contracts.""" + +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +import json + +import diskgraph +import numpy as np +import pyarrow as pa +import pytest + + +class ArrowOnly: + """Expose buffers but fail if the binding falls back to Python iteration.""" + + def __init__(self, array): + self.array = array + + def __arrow_c_array__(self, requested_schema=None): + return self.array.__arrow_c_array__(requested_schema) + + def __iter__(self): + raise AssertionError("input was expanded into Python scalars") + + +def collect(events): + batches = defaultdict(list) + for event in events: + if event["type"] != "stats": + assert isinstance(event["batch"], pa.RecordBatch) + key = (event["type"], event.get("rel_id", event.get("type_id"))) + batches[key].append(event["batch"]) + return {key: pa.Table.from_batches(parts) for key, parts in batches.items()} + + +def assert_batches_equal(actual, expected): + assert actual.keys() == expected.keys() + for key in actual: + assert actual[key].equals(expected[key]), key + + +def sample_plan(): + relations = [ + "orders_parquet_user_id_to_users_user_id_reverse", + "orders_parquet_merchant_id_to_merchants_merchant_id", + ] + return json.dumps({"layers": [ + {"fanouts": {relation: { + "fanout": 10, "mode": "deterministic_uniform", + "absent_key_policy": "uniform_fallback", + }}, "default_fanout": None} + for relation in relations + ]}) + + +@pytest.mark.parametrize("fused", [False, True]) +@pytest.mark.parametrize("swapped", [False, True]) +@pytest.mark.parametrize("dtype", [pa.uint64(), pa.int64()]) +def test_array_sampling_matches_lists_with_repeated_anchors(orders_graph, fused, swapped, dtype): + engine = diskgraph.SamplerEngine(orders_graph, random_seed=41, num_threads=1) + # Same source entity, different anchor time; one root has no neighbors. + keys = pa.array(["u1", "u1", "u3", "u2"]) + ids = engine.lookup_node_ids("users", keys) + times = [1609459200, 1609718400, 1609718400, 1609718400] + options = dict(fused=fused, swap_edge_direction=swapped, + seed_timestamp_unit=diskgraph.TimeUnit.Seconds) + expected = collect(engine.sample( + {"users": ids}, sample_plan(), seed_timestamps={"users": times}, **options)) + # Sliced input buffers must retain the logical offset, not read the prefix. + column = pa.array([None] + ids + [None], type=dtype).slice(1, len(ids)) + cutoffs = pa.array([None] + times + [None], type=pa.int64()).slice(1, len(times)) + actual = collect(engine.sample( + {"users": ArrowOnly(column)}, sample_plan(), + seed_timestamps={"users": ArrowOnly(cutoffs)}, **options)) + assert_batches_equal(actual, expected) + from_keys = collect(engine.sample_from_keys( + {"users": ArrowOnly(keys)}, sample_plan(), + seed_timestamps={"users": ArrowOnly(cutoffs)}, **options)) + assert_batches_equal(from_keys, expected) + # The native Arrow topology can feed NumPy/tensors without Python row objects. + for (kind, _), table in actual.items(): + if kind == "node": + owners = table["id_a"].combine_chunks().to_numpy(zero_copy_only=True) + assert owners.dtype == np.uint64 + assert np.all(owners < len(keys)) + + +@pytest.mark.parametrize("table,keys", [ + ("signed_ids", pa.array([-5, 7, 7], type=pa.int64())), + ("unsigned_ids", pa.array([2**63 + 1, 7], type=pa.uint64())), + ("string_ids", pa.array(["0xAB", " a ", "0xAB"])), +]) +def test_lookup_model_output_and_key_sampling(typed_primary_key_graph, table, keys): + engine = diskgraph.SamplerEngine(typed_primary_key_graph) + expected = engine.lookup_node_ids(table, keys) + ids = engine.lookup_node_ids(table, ArrowOnly(keys), output_format="model") + assert ids.type == pa.uint64() + assert ids.to_pylist() == expected + assert engine.canonical_node_mask(table, ids, keys) == [True] * len(keys) + plan = json.dumps({"layers": []}) + assert_batches_equal( + collect(engine.sample_from_keys({table: keys}, plan, fused=True)), + collect(engine.sample({table: expected}, plan, fused=True)), + ) + + +def test_lookup_missing_and_empty(orders_graph): + engine = diskgraph.SamplerEngine(orders_graph) + keys = pa.array(["u1", "missing", "u1"]) + ids = engine.lookup_node_ids("users", keys, output_format="model") + assert ids.type == pa.uint64() + assert ids.to_pylist() == engine.lookup_node_ids("users", keys) + assert ids.null_count == 1 + empty = engine.lookup_node_ids("users", pa.array([], type=pa.string()), output_format="model") + assert empty.type == pa.uint64() and len(empty) == 0 + with pytest.raises(diskgraph.DiskgraphDataError, match="batch offset 1"): + engine.sample_from_keys({"users": keys}, sample_plan(), fused=True) + with pytest.raises(diskgraph.DiskgraphDataError, match="null"): + engine.sample({"users": ids}, sample_plan(), fused=True) + with pytest.raises(diskgraph.DiskgraphConfigError, match="output_format"): + engine.lookup_node_ids("users", keys, output_format="invalid") + + +@pytest.mark.parametrize("ids,error,match", [ + (pa.array([0, None], type=pa.uint64()), diskgraph.DiskgraphDataError, "null"), + (pa.array([-1], type=pa.int64()), diskgraph.DiskgraphDataError, "negative"), + (pa.array([0.0]), TypeError, "uint64 or int64"), + (pa.array(["0"]), TypeError, "uint64 or int64"), + (pa.chunked_array([[0], [1]], type=pa.uint64()), TypeError, "combine chunks"), +]) +def test_invalid_id_columns_fail_before_sampling(orders_graph, ids, error, match): + engine = diskgraph.SamplerEngine(orders_graph) + with pytest.raises(error, match=match): + engine.sample({"users": ids}, sample_plan(), fused=True) + with pytest.raises(error, match=match): + engine.fetch_features([{"node_type": "users", "node_ids": ids, "feature_names": []}]) + + +@pytest.mark.parametrize("times,error,match", [ + (pa.array([None], type=pa.int64()), diskgraph.DiskgraphDataError, "null"), + (pa.array([1.5]), TypeError, "int64"), + (pa.array([1], type=pa.timestamp("s")), TypeError, "declared time unit"), + (pa.array([], type=pa.int64()), diskgraph.DiskgraphConfigError, "equal length"), + ([], diskgraph.DiskgraphConfigError, "equal length"), +]) +def test_invalid_cutoffs_fail_before_sampling(orders_graph, times, error, match): + engine = diskgraph.SamplerEngine(orders_graph) + with pytest.raises(error, match=match): + engine.sample({"users": [0]}, sample_plan(), fused=True, + seed_timestamps={"users": times}) + with pytest.raises(error, match=match): + engine.sample_from_keys({"users": pa.array(["u1"])}, sample_plan(), fused=True, + seed_timestamps={"users": times}) + + +def test_unknown_timestamp_type_and_invalid_keys(orders_graph): + engine = diskgraph.SamplerEngine(orders_graph) + with pytest.raises(diskgraph.DiskgraphConfigError, match="matching seed type"): + engine.sample({"users": [0]}, sample_plan(), fused=True, + seed_timestamps={"not_users": [0]}) + with pytest.raises(diskgraph.DiskgraphConfigError, match="unknown node type"): + engine.sample_from_keys({"not_users": pa.array(["u1"])}, sample_plan(), fused=True) + with pytest.raises(diskgraph.DiskgraphDataError, match="null primary key"): + engine.sample_from_keys({"users": pa.array([None], type=pa.string())}, sample_plan(), fused=True) + + +def test_array_feature_fetch_preserves_projection_duplicates_and_lifetime(orders_graph): + engine = diskgraph.SamplerEngine(orders_graph) + ids = engine.lookup_node_ids("users", pa.array(["u2", "u1", "u2"])) + expected = collect(engine.fetch_features([ + {"node_type": "users", "node_ids": ids, "feature_names": ["age"]}])) + iterator = engine.fetch_features([ + {"node_type": "users", "node_ids": ArrowOnly(pa.array(ids, type=pa.uint64())), + "feature_names": ["age"]}]) + # The async request owns its imported data even after the Python engine is gone. + del engine + actual = collect(iterator) + assert_batches_equal(actual, expected) + (table,) = actual.values() + assert table["age"].to_pylist() == [30, 25, 30] + + +def temporal_plan(): + return json.dumps({ + "frames": [{"name": "orders", "root_type": "users", "path": [{ + "relation": "orders_parquet_user_id_to_users_user_id_reverse", + "destination_type": "orders_parquet", + }], "time_column": "timestamp", "stop_seconds": 0, "distinct": True, + "columns": ["amount", "order_id"]}], + "sequences": [{"name": "recent", "frame": "orders", "columns": ["amount"], + "order": "descending", "limit": 2}], + "aggregates": [{"name": "total", "frame": "orders", "operation": "sum", "column": "amount"}], + }) + + +@pytest.mark.parametrize("empty", [False, True]) +def test_temporal_arrays_match_lists_and_lengths(sorted_orders_graph, empty): + engine = diskgraph.SamplerEngine(sorted_orders_graph) + keys = pa.array([] if empty else ["u1", "u1", "u2", "u3"], type=pa.string()) + ids = engine.lookup_node_ids("users", keys, output_format="model") + times = pa.array([] if empty else [1609459200, 1609718400, 1609718400, 1609718400], type=pa.int64()) + expected = engine.query_temporal({"users": ids.to_pylist()}, {"users": times.to_pylist()}, + temporal_plan(), output_format="model") + actual = engine.query_temporal({"users": ArrowOnly(ids)}, {"users": ArrowOnly(times)}, + temporal_plan(), output_format="model") + for got, want in zip(actual["sequences"], expected["sequences"]): + for key in ("batch", "offsets", "node_ids", "timestamps"): + assert got[key].equals(want[key]) + for got, want in zip(actual["aggregates"], expected["aggregates"]): + assert got["values"].equals(want["values"]) + lengths = engine.query_temporal_lengths({"users": ids}, {"users": times}, temporal_plan()) + expected_lengths = np.diff(actual["sequences"][0]["offsets"].to_numpy()) + np.testing.assert_array_equal(lengths["sequences"][0]["lengths"].to_numpy(), expected_lengths) + + +def test_empty_sampling_and_concurrent_calls(orders_graph): + engine = diskgraph.SamplerEngine(orders_graph, random_seed=41, num_threads=1) + empty = collect(engine.sample_from_keys({"users": pa.array([], type=pa.string())}, + sample_plan(), fused=True)) + assert not empty + keys = pa.array(["u2", "u1", "u1"]) + expected = collect(engine.sample_from_keys({"users": keys}, sample_plan(), fused=True)) + def run(_): + return collect(engine.sample_from_keys({"users": ArrowOnly(keys)}, sample_plan(), fused=True)) + with ThreadPoolExecutor(max_workers=3) as pool: + for actual in pool.map(run, range(6)): + assert_batches_equal(actual, expected) diff --git a/packages/diskgraph/diskgraph-pybind/src/columnar_input.rs b/packages/diskgraph/diskgraph-pybind/src/columnar_input.rs new file mode 100644 index 000000000..38df24fcc --- /dev/null +++ b/packages/diskgraph/diskgraph-pybind/src/columnar_input.rs @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +// All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Import batch columns without iterating over Python scalars. The engine owns +//! Vec inputs, so this removes Python boxing, not the native buffer copy. + +use arrow::array::{make_array, Array, ArrayData, ArrayRef, Int64Array, UInt64Array}; +use arrow::datatypes::DataType; +use arrow::pyarrow::FromPyArrow; +use pyo3::exceptions::PyTypeError; +use pyo3::prelude::*; +use pyo3::types::PyDict; +use std::collections::HashMap; + +use crate::{DiskgraphConfigError, DiskgraphDataError}; + +fn array_input(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + if value.hasattr("__arrow_c_array__")? || value.hasattr("_export_to_c")? { + let array = make_array(ArrayData::from_pyarrow_bound(value)?); + if array.null_count() != 0 { + return Err(DiskgraphDataError::new_err(format!( + "{name} must not contain nulls" + ))); + } + Ok(Some(array)) + } else if value.hasattr("__arrow_c_stream__")? { + Err(PyTypeError::new_err(format!( + "{name} requires one Arrow Array; combine chunks explicitly" + ))) + } else { + Ok(None) + } +} + +pub(crate) fn node_ids(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let Some(array) = array_input(value, name)? else { + return value.extract(); + }; + match array.data_type() { + DataType::UInt64 => Ok(array + .as_any() + .downcast_ref::() + .expect("checked uint64") + .values() + .to_vec()), + DataType::Int64 => array + .as_any() + .downcast_ref::() + .expect("checked int64") + .values() + .iter() + .enumerate() + .map(|(offset, &id)| { + u64::try_from(id).map_err(|_| { + DiskgraphDataError::new_err(format!( + "{name} has negative node ID at batch offset {offset}" + )) + }) + }) + .collect(), + dtype => Err(PyTypeError::new_err(format!( + "{name} requires Arrow uint64 or int64, got {dtype}" + ))), + } +} + +pub(crate) fn timestamps(value: &Bound<'_, PyAny>, name: &str) -> PyResult> { + let Some(array) = array_input(value, name)? else { + return value.extract(); + }; + let values = array.as_any().downcast_ref::().ok_or_else(|| { + PyTypeError::new_err(format!( + "{name} requires Arrow int64 in the API's declared time unit, got {}", + array.data_type() + )) + })?; + Ok(values.values().to_vec()) +} + +pub(crate) fn timestamp_map( + values: Option>, + seeds: &HashMap>, +) -> PyResult>>> { + let Some(values) = values else { + return Ok(None); + }; + let mut result = HashMap::new(); + for (key, value) in values.iter() { + let node_type: String = key.extract()?; + let ids = seeds.get(&node_type).ok_or_else(|| { + DiskgraphConfigError::new_err(format!( + "seed_timestamps has no matching seed type {node_type:?}" + )) + })?; + let times = timestamps(&value, "seed_timestamps")?; + if times.len() != ids.len() { + return Err(DiskgraphConfigError::new_err(format!( + "seed_timestamps for {node_type:?} must have equal length to seeds: {} != {}", + times.len(), + ids.len() + ))); + } + result.insert(node_type, times); + } + Ok(Some(result)) +} diff --git a/packages/diskgraph/diskgraph-pybind/src/lib.rs b/packages/diskgraph/diskgraph-pybind/src/lib.rs index 341ef79fd..278698e80 100644 --- a/packages/diskgraph/diskgraph-pybind/src/lib.rs +++ b/packages/diskgraph/diskgraph-pybind/src/lib.rs @@ -187,6 +187,7 @@ mod artifact_helpers; mod bp_col_stats; mod bp_evaluate; mod col_stats_export; +mod columnar_input; mod cutoff_col_stats; mod custom_tt_contract; mod custom_tt_entry; @@ -739,6 +740,137 @@ fn lookup_node_ids_from_arrow( primary_key_lookup_from_arrow(table, values, |_, node_id| node_id) } +impl SamplerEngine { + fn resolve_keys( + &self, + py: Python<'_>, + node_type: &str, + primary_keys: &Bound<'_, PyAny>, + ) -> PyResult>> { + let primary_keys = make_array(ArrayData::from_pyarrow_bound(primary_keys)?); + let table = self.reader.table(node_type).ok_or_else(|| { + DiskgraphConfigError::new_err(format!("unknown node type: {node_type}")) + })?; + if table.pk_map().is_none() { + return Err(DiskgraphConfigError::new_err(format!( + "node type {node_type} has no primary-key index" + ))); + } + py.detach(move || lookup_node_ids_from_arrow(&table, primary_keys.as_ref())) + } + + fn start_sample( + &self, + py: Python<'_>, + seeds: HashMap>, + plan_json: String, + fused: bool, + timestamps: Option>>, + swap_edge_direction: Option, + seed_timestamp_unit: Option, + ) -> PyResult { + let plan: SamplingPlan = serde_json::from_str(&plan_json) + .map_err(|e| DiskgraphConfigError::new_err(format!("Invalid plan JSON: {}", e)))?; + + let flags = SamplingFlags { + replace: false, + directed: true, + disjoint: true, + return_edge_id: false, + adaptive: false, + fused_feature_fetching: fused, + swap_edge_direction: swap_edge_direction.unwrap_or(true), + }; + + let parsed_unit = seed_timestamp_unit + .map(|u| u.into()) + .unwrap_or(diskgraph_core::time::TimeUnit::Nanoseconds); + + let request = BatchSamplingRequest { + dataset_id: uuid::Uuid::nil(), + seed_nodes: seeds, + seed_timestamps: timestamps, + seed_timestamp_unit: parsed_unit, + plan, + flags, + }; + + let seed_type = request + .seed_nodes + .keys() + .next() + .cloned() + .unwrap_or_default(); + let seed_count = request + .seed_nodes + .values() + .next() + .map_or(0usize, |v| v.len()); + + let count = self + .sample_count + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if count % 100 == 0 { + info!( + seed_type = %seed_type, + seed_count, + hops = request.plan.layers.len(), + fused_feature_fetching = request.flags.fused_feature_fetching, + swap_edge_direction = request.flags.swap_edge_direction, + "Pybind sampling request started" + ); + } else { + tracing::debug!( + seed_type = %seed_type, + seed_count, + hops = request.plan.layers.len(), + fused_feature_fetching = request.flags.fused_feature_fetching, + swap_edge_direction = request.flags.swap_edge_direction, + "Pybind sampling request started" + ); + } + + let (tx, rx) = bounded(1); + let engine = Arc::clone(&self.engine); + let mut state = self.pool.acquire(); + + py.detach(move || { + thread::spawn(move || { + let mut handler = PyOutputHandler::new( + tx.clone(), + engine.relation_src_type.clone(), + engine.relation_info.clone(), + ); + match engine.sample(&request, &mut state, &mut handler) { + Ok(()) => { + if count % 100 == 0 { + info!( + total_nodes = state.stats.total_nodes, + total_edges = state.stats.total_edges, + process_rss_mb = get_current_rss_mb(), + "Pybind sampling request finished" + ); + } else { + tracing::debug!( + total_nodes = state.stats.total_nodes, + total_edges = state.stats.total_edges, + process_rss_mb = get_current_rss_mb(), + "Pybind sampling request finished" + ); + } + } + Err(e) => { + error!(error = %e, "Pybind sampling request failed"); + let _ = tx.send(SampleEvent::Error { err: e }); + } + } + }); + }); + + Ok(PySampleIterator { receiver: rx }) + } +} + #[pymethods] impl SamplerEngine { #[new] @@ -767,22 +899,25 @@ impl SamplerEngine { /// Missing keys are returned as ``None`` in their original positions. A /// null or physically incompatible Arrow value is a data error rather than /// being conflated with a well-formed key that is absent from the index. + #[pyo3(signature = (node_type, primary_keys, *, output_format = "inspectable"))] pub fn lookup_node_ids( &self, py: Python<'_>, node_type: String, primary_keys: &Bound<'_, PyAny>, - ) -> PyResult>> { - let primary_keys = make_array(ArrayData::from_pyarrow_bound(primary_keys)?); - let table = self.reader.table(&node_type).ok_or_else(|| { - DiskgraphConfigError::new_err(format!("unknown node type: {node_type}")) - })?; - if table.pk_map().is_none() { + output_format: &str, + ) -> PyResult> { + if !matches!(output_format, "inspectable" | "model") { return Err(DiskgraphConfigError::new_err(format!( - "node type {node_type} has no primary-key index" + "output_format must be 'inspectable' or 'model', got {output_format:?}" ))); } - py.detach(move || lookup_node_ids_from_arrow(&table, primary_keys.as_ref())) + let ids = self.resolve_keys(py, &node_type, primary_keys)?; + if output_format == "model" { + Ok(UInt64Array::from(ids).to_data().to_pyarrow(py)?.unbind()) + } else { + Ok(ids.into_pyobject(py)?.into_any().unbind()) + } } /// Traverse exact relation paths and derive temporal sequences/aggregates. @@ -820,13 +955,13 @@ impl SamplerEngine { .next() .expect("one seed node entry was validated"); let root_type: String = root_key.extract()?; - let roots: Vec = root_values.extract()?; + let roots = columnar_input::node_ids(&root_values, "seed_nodes")?; let cutoff_values = seed_timestamps.get_item(&root_type)?.ok_or_else(|| { DiskgraphConfigError::new_err(format!( "seed_timestamps has no entry for root type {root_type:?}" )) })?; - let cutoffs: Vec = cutoff_values.extract()?; + let cutoffs = columnar_input::timestamps(&cutoff_values, "seed_timestamps")?; let reader = Arc::clone(&self.reader); let engine = Arc::clone(&self.engine); let result = py @@ -941,13 +1076,13 @@ impl SamplerEngine { .next() .expect("one seed node entry was validated"); let root_type: String = root_key.extract()?; - let roots: Vec = root_values.extract()?; + let roots = columnar_input::node_ids(&root_values, "seed_nodes")?; let cutoff_values = seed_timestamps.get_item(&root_type)?.ok_or_else(|| { DiskgraphConfigError::new_err(format!( "seed_timestamps has no entry for root type {root_type:?}" )) })?; - let cutoffs: Vec = cutoff_values.extract()?; + let cutoffs = columnar_input::timestamps(&cutoff_values, "seed_timestamps")?; let reader = Arc::clone(&self.reader); let engine = Arc::clone(&self.engine); let result = py @@ -1009,120 +1144,64 @@ impl SamplerEngine { let mut seeds = HashMap::new(); for (key, value) in seed_nodes.into_iter() { let type_name: String = key.extract()?; - let ids: Vec = value.extract()?; + let ids = columnar_input::node_ids(&value, "seed_nodes")?; seeds.insert(type_name, ids); } - let mut timestamps = None; - if let Some(st) = seed_timestamps { - let mut ts_map = HashMap::new(); - for (key, value) in st.into_iter() { - let type_name: String = key.extract()?; - let ts: Vec = value.extract()?; - ts_map.insert(type_name, ts); - } - timestamps = Some(ts_map); - } - - let plan: SamplingPlan = serde_json::from_str(&plan_json) - .map_err(|e| DiskgraphConfigError::new_err(format!("Invalid plan JSON: {}", e)))?; - - let flags = SamplingFlags { - replace: false, - directed: true, - disjoint: true, - return_edge_id: false, - adaptive: false, - fused_feature_fetching: fused, - swap_edge_direction: swap_edge_direction.unwrap_or(true), - }; - - let parsed_unit = seed_timestamp_unit - .map(|u| u.into()) - .unwrap_or(diskgraph_core::time::TimeUnit::Nanoseconds); - - let request = BatchSamplingRequest { - dataset_id: uuid::Uuid::nil(), - seed_nodes: seeds, - seed_timestamps: timestamps, - seed_timestamp_unit: parsed_unit, - plan, - flags, - }; + let timestamps = columnar_input::timestamp_map(seed_timestamps, &seeds)?; - let seed_type = request - .seed_nodes - .keys() - .next() - .cloned() - .unwrap_or_default(); - let seed_count = request - .seed_nodes - .values() - .next() - .map_or(0usize, |v| v.len()); + self.start_sample( + py, + seeds, + plan_json, + fused, + timestamps, + swap_edge_direction, + seed_timestamp_unit, + ) + } - let count = self - .sample_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - if count % 100 == 0 { - info!( - seed_type = %seed_type, - seed_count, - hops = request.plan.layers.len(), - fused_feature_fetching = request.flags.fused_feature_fetching, - swap_edge_direction = request.flags.swap_edge_direction, - "Pybind sampling request started" - ); - } else { - tracing::debug!( - seed_type = %seed_type, - seed_count, - hops = request.plan.layers.len(), - fused_feature_fetching = request.flags.fused_feature_fetching, - swap_edge_direction = request.flags.swap_edge_direction, - "Pybind sampling request started" - ); + /// Resolve typed raw keys and sample in one call. Missing keys fail before + /// sampling; dropping them would change per-anchor ownership and cutoffs. + /// With `fused=True`, the existing sampler also returns feature batches. + #[pyo3(signature = (seed_primary_keys, plan_json, fused, seed_timestamps=None, swap_edge_direction=None, seed_timestamp_unit=None))] + pub fn sample_from_keys( + &self, + py: Python<'_>, + seed_primary_keys: Bound<'_, PyDict>, + plan_json: String, + fused: bool, + seed_timestamps: Option>, + swap_edge_direction: Option, + seed_timestamp_unit: Option, + ) -> PyResult { + let mut seeds = HashMap::new(); + for (key, values) in seed_primary_keys.iter() { + let node_type: String = key.extract()?; + let ids = self.resolve_keys(py, &node_type, &values)?; + let ids = ids + .into_iter() + .enumerate() + .map(|(offset, id)| { + id.ok_or_else(|| { + DiskgraphDataError::new_err(format!( + "missing primary key for {node_type:?} at batch offset {offset}" + )) + }) + }) + .collect::>>()?; + seeds.insert(node_type, ids); } - - let (tx, rx) = bounded(1); - let engine = Arc::clone(&self.engine); - let mut state = self.pool.acquire(); - - py.detach(move || { - thread::spawn(move || { - let mut handler = PyOutputHandler::new( - tx.clone(), - engine.relation_src_type.clone(), - engine.relation_info.clone(), - ); - match engine.sample(&request, &mut state, &mut handler) { - Ok(()) => { - if count % 100 == 0 { - info!( - total_nodes = state.stats.total_nodes, - total_edges = state.stats.total_edges, - process_rss_mb = get_current_rss_mb(), - "Pybind sampling request finished" - ); - } else { - tracing::debug!( - total_nodes = state.stats.total_nodes, - total_edges = state.stats.total_edges, - process_rss_mb = get_current_rss_mb(), - "Pybind sampling request finished" - ); - } - } - Err(e) => { - error!(error = %e, "Pybind sampling request failed"); - let _ = tx.send(SampleEvent::Error { err: e }); - } - } - }); - }); - - Ok(PySampleIterator { receiver: rx }) + let timestamps = columnar_input::timestamp_map(seed_timestamps, &seeds)?; + self.start_sample( + py, + seeds, + plan_json, + fused, + timestamps, + swap_edge_direction, + seed_timestamp_unit, + ) } #[tracing::instrument(level = "debug", skip(self, py, batch))] @@ -1140,12 +1219,12 @@ impl SamplerEngine { pyo3::exceptions::PyKeyError::new_err("missing 'node_type' in feature request") })? .extract()?; - let node_ids: Vec = dict - .get_item("node_ids")? - .ok_or_else(|| { + let node_ids = columnar_input::node_ids( + &dict.get_item("node_ids")?.ok_or_else(|| { pyo3::exceptions::PyKeyError::new_err("missing 'node_ids' in feature request") - })? - .extract()?; + })?, + "node_ids", + )?; let feature_names: Vec = dict .get_item("feature_names")? .ok_or_else(|| {