diff --git a/.github/workflows/kumo-fm.yml b/.github/workflows/kumo-fm.yml new file mode 100644 index 000000000..28a9e6a21 --- /dev/null +++ b/.github/workflows/kumo-fm.yml @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +name: kumo-fm + +on: # yamllint disable-line rule:truthy + pull_request: + paths: + - 'packages/experimental/kumo-fm/**' + - 'packages/experimental/kumo-pytorch/**' + - 'packages/diskgraph/**' + - 'uv.lock' + - '.github/workflows/kumo-fm.yml' + push: + branches: [main] + paths: + - 'packages/experimental/kumo-fm/**' + - 'packages/experimental/kumo-pytorch/**' + - 'packages/diskgraph/**' + - 'uv.lock' + - '.github/workflows/kumo-fm.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + diskgraph-wheel: + uses: ./.github/workflows/_diskgraph-wheel.yml + + package-and-wheel: + needs: diskgraph-wheel + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: ./.github/actions/kumo-pytorch-setup + with: + diskgraph-wheel-artifact: diskgraph-wheel + - name: Test the source package against the current native stack + run: | + uv sync --locked --inexact --no-install-package diskgraph --no-install-package torch \ + --package kumo-fm --extra test + uv run --no-sync --package kumo-fm python -m pytest -q \ + packages/experimental/kumo-fm/tests + - name: Build both finished wheels + run: | + uv build --wheel --package kumo-pytorch --out-dir "$RUNNER_TEMP/dist" + uv build --sdist --package kumo-fm --out-dir "$RUNNER_TEMP/dist" + uv build --wheel "$RUNNER_TEMP"/dist/kumo_fm-*.tar.gz --out-dir "$RUNNER_TEMP/dist" + - name: Install finished wheels outside the checkout + working-directory: ${{ runner.temp }} + run: | + uv venv --python 3.12 fresh + uv pip install --python fresh/bin/python --torch-backend cpu \ + /tmp/diskgraph-wheel/diskgraph-*.whl dist/kumo_pytorch-*.whl \ + dist/kumo_fm-*.whl 'pytest>=8,<10' + uv pip check --python fresh/bin/python + - name: Run tests using only installed runtime packages + run: | + mkdir "$RUNNER_TEMP/stage" + cp -R packages/experimental/kumo-fm/tests \ + packages/experimental/kumo-fm/examples "$RUNNER_TEMP/stage/" + cd "$RUNNER_TEMP/stage" + ../fresh/bin/python -I -c 'import kumo.fm, kumo.pytorch, diskgraph; print(kumo.fm.__file__)' + ../fresh/bin/python -m pytest -q tests + - name: Build matching connector and engine wheels + run: | + for package in kumo-api kumo-pql kumo-settings kumo-core kumo-engine; do + uv build --wheel --package "$package" --out-dir "$RUNNER_TEMP/engine-wheels" + done + - name: Test connector snapshot to FM from installed wheels + working-directory: ${{ runner.temp }}/stage + run: | + uv pip install --python ../fresh/bin/python --torch-backend cpu \ + ../engine-wheels/*.whl + uv pip check --python ../fresh/bin/python + ../fresh/bin/python -I -c 'import kumo.engine, kumo.fm, kumo.pytorch, diskgraph' + ../fresh/bin/python -m pytest -q tests/test_connector_snapshot.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ba3fab5af..eddcf7445 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -12,6 +12,7 @@ This monorepo contains Kumo's graph machine-learning application, Python package | Service Python SDK | `packages/kumo-sdk/` | Python client for a configured Kumo service (`kumoai_studio`) | | Local Python SDK | `packages/kumo-engine/` | In-process SDK surface (`kumo.engine`) for local execution with local or remote data sources | | Shared runtime | `packages/kumo-core/` | Service-independent data processing, training, evaluation, and prediction used by the service and local SDK | +| Experimental model data | `packages/experimental/kumo-pytorch/`, `packages/experimental/kumo-fm/` | Public columnar sampling and FM-specific Arrow payload assembly | | Connectors | `kumo/connector/` | Data-warehouse and file-source integrations | | Data pipeline | `kumo/data/` | Export, cache, and DiskGraph-backed table generation | | Query language | `packages/kumo-pql/`, `kumo/pquery/` | Parser and semantic validation package; service query and table-generation orchestration | diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md new file mode 100644 index 000000000..7f2b42d3b --- /dev/null +++ b/benchmark/fm-payloads/README.md @@ -0,0 +1,44 @@ +# FM payload comparison + +`runner.py` compares the installed `kumo-fm` wheel with the pinned +fm-dataloader DuckDB reference on the same frozen bundle, workload, and +selected instances. Install the reference checkout only in the benchmark +environment; it is not a `kumo-fm` runtime dependency. + +Run correctness first, then warmups and interleaved measured draws: + +```bash +python make_fixture.py /tmp/fm-case +python runner.py --bundle /tmp/fm-case/bundle \ + --workload /tmp/fm-case/workload.json --index /tmp/fm-case/index \ + --draws 5 --output /tmp/fm-report.json +``` + +For an existing bundle and workload: + +```bash +python runner.py --bundle /data/bundle --workload /data/workload.json \ + --index /data/fm-index --draws 7 --output /tmp/fm-report.json +``` + +The workload must be eligible for the public `kumo.pytorch` recent-sampling +path. Uniform/timeless and empty-layer workloads are reported as excluded, +not silently rewritten. Reports include source revisions, dependency +versions, machine metadata, row counts, logical Arrow bytes, body bytes, and +raw per-phase timings. Retain the bundle, workload, wheel hashes, index +provenance, and report together when publishing a comparison. + +## Omnistation smoke comparison + +[Raw report](reports/omnistation-smoke.json) records five interleaved draws +after two warmups on a four-event, one-hop synthetic graph. Both encodings +decoded identically to the DuckDB reference and produced the same body byte +counts. Median phase times in milliseconds: + +| Encoding | DuckDB reference full payload | DiskGraph load and assembly | Encoding retained batch | +| --- | ---: | ---: | ---: | +| Arrays | 40.200 | 3.453 | 0.710 | +| Arrow IPC | 40.141 | 3.477 | 0.424 | + +The backend paths and measured phase boundaries differ. This small smoke +case validates parity and timing instrumentation, not general throughput. diff --git a/benchmark/fm-payloads/make_fixture.py b/benchmark/fm-payloads/make_fixture.py new file mode 100644 index 000000000..08671bc28 --- /dev/null +++ b/benchmark/fm-payloads/make_fixture.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create the frozen smoke case used by the benchmark and SDM notebook.""" + +import json +import sys +from pathlib import Path + +TESTS = Path(__file__).resolve().parents[2] / "packages/experimental/kumo-fm/tests" +sys.path.insert(0, str(TESTS)) + +from test_integration import fixture # noqa: E402 + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: python make_fixture.py /path/to/new/case") + root = Path(sys.argv[1]).resolve() + root.mkdir(parents=True, exist_ok=False) + _, workload = fixture(root) + (root / "supported").rename(root / "bundle") + (root / "workload.json").write_text(json.dumps(workload, indent=2) + "\n") + print(f"bundle: {root / 'bundle'}") + print(f"workload: {root / 'workload.json'}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/fm-payloads/reports/omnistation-smoke.json b/benchmark/fm-payloads/reports/omnistation-smoke.json new file mode 100644 index 000000000..db8081a16 --- /dev/null +++ b/benchmark/fm-payloads/reports/omnistation-smoke.json @@ -0,0 +1,140 @@ +{ + "bundle_manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", + "workload_sha256": "5c479c8c9438f297c23e5f8e0ce00ff95fc406f36bfe765c94723f75dbed0395", + "wheel_sha256": { + "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "92cd45fb8691db74c86ab53f382a2880231008f2db02ba1de4ae77a259479eb6", + "diskgraph-0.2.0-cp311-abi3-linux_x86_64.whl": "ede0ca14dee5ccc75266afaf55c8ffd90f73a6e6f835448ddeff87655186522e" + }, + "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", + "engine_revision": "539e9a4f1cb7ae7cfa254ea2eac90629a07a032b", + "reference_revision": "b4516394e98929c5b4752745866d5e3138276255", + "checkout_dirty": false, + "machine": { + "platform": "Linux-7.0.0-1010-aws-x86_64-with-glibc2.39", + "processor": "x86_64", + "python": "3.11.15" + }, + "versions": { + "kumo-fm": "0.1.0a1", + "kumo-pytorch": "0.1.0a1", + "diskgraph": "0.2.0", + "fm-dataloader": "0.1.0", + "duckdb": "2.0.0", + "pyarrow": "24.0.0", + "numpy": "2.4.6" + }, + "cases": [ + { + "encoding": "arrays", + "context_rows": 2, + "predict_rows": 4, + "logical_arrow_bytes": 584, + "new_body_bytes": 3374, + "reference_body_bytes": 3374, + "draws": [ + { + "reference_full_ms": 41.671234, + "new_load_ms": 3.360692, + "new_encode_ms": 0.694896 + }, + { + "reference_full_ms": 39.565126, + "new_load_ms": 3.484977, + "new_encode_ms": 0.713286 + }, + { + "reference_full_ms": 40.508425, + "new_load_ms": 3.447152, + "new_encode_ms": 0.710074 + }, + { + "reference_full_ms": 40.103816, + "new_load_ms": 3.452723, + "new_encode_ms": 0.979746 + }, + { + "reference_full_ms": 40.199899, + "new_load_ms": 3.533769, + "new_encode_ms": 0.698663 + } + ] + }, + { + "encoding": "arrow_ipc", + "context_rows": 2, + "predict_rows": 4, + "logical_arrow_bytes": 584, + "new_body_bytes": 8859, + "reference_body_bytes": 8859, + "draws": [ + { + "reference_full_ms": 40.14074, + "new_load_ms": 3.622619, + "new_encode_ms": 0.465381 + }, + { + "reference_full_ms": 41.85428, + "new_load_ms": 3.477252, + "new_encode_ms": 0.424267 + }, + { + "reference_full_ms": 40.112547, + "new_load_ms": 3.372056, + "new_encode_ms": 0.444124 + }, + { + "reference_full_ms": 41.194207, + "new_load_ms": 3.428842, + "new_encode_ms": 0.422983 + }, + { + "reference_full_ms": 40.038979, + "new_load_ms": 3.915465, + "new_encode_ms": 0.420586 + } + ] + } + ], + "warmups": 2, + "draws_per_encoding": 5, + "graph": { + "layers": [ + [ + { + "relation": "interaction_user", + "direction": "reverse", + "fanout": 1 + } + ] + ], + "table_rows": { + "interactions": 4, + "users": 3 + }, + "relations": [ + "interaction_user" + ] + }, + "eligibility": "supported", + "exclusion_reasons": [], + "index_marker_sha256": "0ada8dd747a380a67199e9d6138f02178a4f926c2a90ffebea9468fab2d4f087", + "selected_rows": { + "context": { + "interactions": 2, + "users": 2 + }, + "predict": { + "interactions": 3, + "users": 4 + } + }, + "selected_edges": { + "context": { + "interaction_user": 2 + }, + "predict": { + "interaction_user": 3 + } + }, + "correctness": "passed" +} diff --git a/benchmark/fm-payloads/runner.py b/benchmark/fm-payloads/runner.py new file mode 100644 index 000000000..c150c9d35 --- /dev/null +++ b/benchmark/fm-payloads/runner.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Correctness-gated, interleaved FM payload timing against the DuckDB reference.""" +import argparse +import hashlib +import importlib.metadata +import json +import platform +import random +import subprocess +import time +from pathlib import Path +from urllib.parse import urlparse + +from kumo import fm +from kumo.fm._pytorch import compatibility_issues +from kumo.fm.bundle import Bundle +from kumo.fm.workload import Workload + + +def revision(path): + result = subprocess.run(["git", "rev-parse", "HEAD"], cwd=path, capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def installed_revision(name): + distribution = importlib.metadata.distribution(name) + source = json.loads(distribution.read_text("direct_url.json") or "{}").get("url") + return revision(Path(urlparse(source).path)) if source and source.startswith("file://") else None + + +def measure(callback): + start = time.perf_counter_ns() + result = callback() + return result, (time.perf_counter_ns() - start) / 1e6 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--workload", type=Path, required=True) + parser.add_argument("--index", type=Path, required=True) + parser.add_argument("--draws", type=int, default=7) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--wheel", type=Path, action="append", default=[]) + args = parser.parse_args() + if args.draws < 1 or args.warmups < 1: + parser.error("draws and warmups must be positive") + + from fm_dataloader.bundle import Bundle as OldBundle + from fm_dataloader.multihop_sql import DuckDBMultihopBackend + from fm_dataloader.payload import build_payload, decode_payload + from fm_dataloader.workload import Workload as OldWorkload + + old_bundle = OldBundle(args.bundle) + old_workload = OldWorkload.load(old_bundle, args.workload) + new_bundle = Bundle(args.bundle) + new_workload = Workload.load(new_bundle, args.workload) + issues = compatibility_issues(new_workload) + report = { + "bundle_manifest_sha256": digest(args.bundle / "manifest.json"), + "workload_sha256": digest(args.workload), + "wheel_sha256": {path.name: digest(path) for path in args.wheel}, + "snapshot_id": new_bundle.snapshot_id, + "engine_revision": revision(Path(__file__).resolve().parents[2]), + "reference_revision": installed_revision("fm-dataloader"), + "checkout_dirty": bool(subprocess.run( + ["git", "status", "--porcelain"], cwd=Path(__file__).resolve().parents[2], + capture_output=True, text=True).stdout.strip()), + "machine": {"platform": platform.platform(), "processor": platform.processor(), + "python": platform.python_version()}, + "versions": {name: importlib.metadata.version(name) for name in + ("kumo-fm", "kumo-pytorch", "diskgraph", "fm-dataloader", + "duckdb", "pyarrow", "numpy")}, + "cases": [], + "warmups": args.warmups, + "draws_per_encoding": args.draws, + "graph": {"layers": new_workload.layers, + "table_rows": {name: new_bundle.tables[name]["rows"] + for name in new_workload.table_names}, + "relations": sorted(new_workload.relation_names)}, + "eligibility": "excluded" if issues else "supported", + "exclusion_reasons": issues, + } + if issues: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + return + old_backend = DuckDBMultihopBackend(old_bundle, native=True) + try: + with fm.prepare(new_bundle, new_workload, index=args.index) as loader: + report["index_marker_sha256"] = digest(args.index / "fm_snapshot.json") + with loader.materialize() as request: + batch = request.load() + report["selected_rows"] = { + split: {name: table.num_rows for name, table in + batch.tables(split)["related_tables"].items()} + for split in batch.splits} + report["selected_edges"] = { + split: { + relation: len(batch.tables(split)["related_tables"][rel["child"]][ + "__fm_edge_" + relation]) + - batch.tables(split)["related_tables"][rel["child"]][ + "__fm_edge_" + relation].null_count + for relation, rel in new_bundle.relations.items() + if relation in new_workload.relation_names} + for split in batch.splits} + for encoding in ("arrays", "arrow_ipc"): + old_body = build_payload(old_backend, old_workload, encoding=encoding) + new_body = batch.encode(encoding) + if decode_payload(old_body) != decode_payload(new_body): + raise AssertionError(f"{encoding} differs from DuckDB reference") + report["cases"].append({ + "encoding": encoding, + "context_rows": batch.tables("context")["instance_table"].num_rows, + "predict_rows": batch.tables("predict")["instance_table"].num_rows, + "logical_arrow_bytes": batch.logical_arrow_bytes, + "new_body_bytes": len(new_body), + "reference_body_bytes": len(old_body), + "draws": [], + }) + for iteration in range(args.warmups + args.draws): + order = report["cases"][:] + random.Random(iteration).shuffle(order) + for case in order: + encoding = case["encoding"] + old_body, old_ms = measure( + lambda: build_payload(old_backend, old_workload, encoding=encoding)) + new_batch, load_ms = measure(request.load) + new_body, encode_ms = measure(lambda: new_batch.encode(encoding)) + if decode_payload(old_body) != decode_payload(new_body): + raise AssertionError(f"{encoding} changed during timing") + if iteration >= args.warmups: + case["draws"].append({ + "reference_full_ms": old_ms, + "new_load_ms": load_ms, + "new_encode_ms": encode_ms, + }) + finally: + old_backend.close() + args.output.parent.mkdir(parents=True, exist_ok=True) + report["correctness"] = "passed" + args.output.write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docs/README.md b/docs/README.md index 80ba64bc9..6dfacafc0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ guides for their interfaces and development workflows. | API schemas | [kumo-api](../packages/kumo-api/README.md) | | Service Python client | [kumo-sdk](../packages/kumo-sdk/README.md) | | In-process Python interface | [kumo-engine](../packages/kumo-engine/README.md) | +| Experimental model batches and FM payloads | [kumo-pytorch](../packages/experimental/kumo-pytorch/README.md) and [kumo-fm](../packages/experimental/kumo-fm/README.md) | | Shared execution runtime | [kumo-core](../packages/kumo-core/README.md) | | Query language | [kumo-pql](../packages/kumo-pql/README.md) and [existing instructions](../packages/kumo-pql/AGENTS.md) | | UI component library | [catalog](../packages/kumo-ui-catalog/README.md) and [existing component guide](../packages/kumo-ui-catalog/CLAUDE.md) | diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 982d7d782..85d75878d 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1241,8 +1241,8 @@ impl GraphIndexReader { // All opening paths share this validation, including direct native // callers and legacy relative references that traverse symlinks. let parent = fs::canonicalize(parent_root).context("resolve parent index path")?; - let subdir = - fs::canonicalize(parent_root.join(subdir_rel)).context("resolve extension index path")?; + let subdir = fs::canonicalize(parent_root.join(subdir_rel)) + .with_context(|| format!("resolve extension `{namespace}` index path: {subdir_rel}"))?; ensure!( subdir != parent, "extension path must not identify the base index" diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index a65da050c..47bbb8aaa 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -414,6 +414,8 @@ def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, stage = parent / "train_tables" / ("base-alias-" + entrypoint) stage.symlink_to(parent, target_is_directory=True) match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension" + if kind == "missing": + match = "extension `stage`" with pytest.raises(diskgraph.DiskgraphSystemError, match=match): if entrypoint == "native": from diskgraph._c import RuntimeIndex as NativeRuntimeIndex diff --git a/packages/experimental/kumo-fm/LICENSE b/packages/experimental/kumo-fm/LICENSE new file mode 100644 index 000000000..d1593aaa1 --- /dev/null +++ b/packages/experimental/kumo-fm/LICENSE @@ -0,0 +1,204 @@ +Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/experimental/kumo-fm/MANIFEST.in b/packages/experimental/kumo-fm/MANIFEST.in new file mode 100644 index 000000000..19ded0427 --- /dev/null +++ b/packages/experimental/kumo-fm/MANIFEST.in @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +recursive-include examples/local_sdm *.py *.ipynb *.md *.lock +recursive-include examples/local_sdm/fixture *.json *.parquet +recursive-include tests *.py diff --git a/packages/experimental/kumo-fm/README.md b/packages/experimental/kumo-fm/README.md new file mode 100644 index 000000000..d97b88cdc --- /dev/null +++ b/packages/experimental/kumo-fm/README.md @@ -0,0 +1,122 @@ +# Kumo FM payload loader + +The primary workflow starts with a graph built through `kumo.engine` connectors. +Adopt its snapshot with `api.DiskGraphEngine.from_snapshot(snapshot, +work_dir=...)`, declare a public PyTorch `SampleSpec` with one +`NeighborhoodSpec` and a bounded future `Aggregate` or `Compare` target, then +call `fm.prepare(engine=..., sample=..., target=..., task=..., features=...)`. +See the runnable +[`connector_to_prediction.py`](examples/local_sdm/connector_to_prediction.py). +FM derives table and relation metadata from `engine.schema`; it creates +context/query anchors, computes context labels in a separate public PyTorch +plan, samples one columnar graph plan over the same index, and aligns both by +occurrence ID. The base index remains unchanged. Columnar graph output does not +currently fuse computed labels into the graph batch. +For an advanced public relation alias or table projection, pass +`schema=api.DataSchema(...)`; PyTorch checks it against the adopted index. + +`fm.Context(entities=..., times=...)` produces entities × times, with time as +the outer axis. `fm.Query(entities=..., at=...)` uses one query time. Either +accepts ordered `pairs=[(entity, time), ...]`; a third element supplies an +explicit int64 occurrence ID. Repeated pairs remain separate. Without explicit +IDs each split uses zero-based positions. Context's future label window must be +fully observable before the earliest query time. Query targets never enter the +request. `loader.materialize(context=..., query=...).load()` and the shortcut +`loader.load(context=..., query=...)` return the same `PayloadBatch` type as +the frozen-bundle path. `batch.tables(split)` and both `batch.encode(...)` +methods reuse its retained Arrow tables without sampling again. + +The snapshot path currently accepts a single public neighborhood, bounded +`recent` fanouts, supported scalar FM wire types and `Aggregate(Rows(..., +time=TimeWindow(start=0, stop=positive)))` or `Compare` over that aggregate. +Multiple `GraphLayer`s in that neighborhood are passed to one native sampling +operation. Classification requires an explicit two-entry raw-label-to-name +`classes` map; regression accepts a numeric aggregate. Unsupported types, +timeless destinations and uniform sampling fail explicitly. Native labels and +graphs remain separate materializations over the adopted snapshot. + +## Frozen bundle compatibility + +Experimental `kumo-fm` builds the existing Kumo Relational HTTP request from +public `kumo.pytorch` columnar graph batches backed by DiskGraph. Sampling and +FM assembly happen once per `PreparedRequest.load()`. The returned +`PayloadBatch` retains typed Arrow tables; calls to `encode("arrays")` and +`encode("arrow_ipc")` serialize those same tables without sampling. + +Build and install `kumo-pytorch` and DiskGraph from the same engine revision, +then build and install this package's wheel. The example under +`examples/local_sdm` additionally needs SDM and a compatible pretrained +checkpoint; SDM is not a runtime dependency. + +```bash +uv build --wheel --package kumo-pytorch --out-dir dist +uv build --sdist --package kumo-fm --out-dir dist +uv build --wheel dist/kumo_fm-*.tar.gz --out-dir dist +uv pip install --python /path/to/venv/bin/python --torch-backend cpu \ + /path/to/matching/diskgraph-*.whl dist/kumo_pytorch-*.whl dist/kumo_fm-*.whl +``` + +```python +from kumo import fm + +with fm.prepare("/data/bundle", "/data/workload.json", index="/data/fm-index") as loader: + with loader.materialize(context=[101, 102], predict=[201]) as request: + batch = request.load() + context = batch.tables("context") + print(context["instance_table"].schema) + arrays_body = batch.encode("arrays") + ipc_body = batch.encode("arrow_ipc") +``` + +The optional `context` and `predict` lists contain ordered instance IDs +already validated by the workload. Omit a list to use that split's frozen +instances. Pass `[]` to omit a split. A direct SDM prediction needs both. +`materialize()` prepares the requested anchors; `load()` samples and assembles +them. Each explicit `load()` can produce a new batch. Retained batches own +their Arrow references after request and loader closure. + +`batch.metadata` and `batch.as_arrow_request()` provide defensive metadata +copies. `batch.tables(split)` returns the instance Arrow table and named +related Arrow tables. `batch.logical_arrow_bytes` sums their `Table.nbytes`; +it is neither peak process memory nor the encoded HTTP body size. Row and +column limits and the decoded Arrow limit apply during assembly. The final +JSON body limit applies only during encoding. Encoding failures leave the +batch usable. + +The wire envelope, composite `(instance_id, primary_key)` identity, selected +`__fm_edge_*` relationship columns, large integer representation, and +context-only target follow the prior loader contract. Arrays remain the +default wire format; Arrow IPC is opt-in. No SDM conversion runs in this wheel. + +## Compatibility + +The production path accepts frozen `native_multihop` workloads whose graph +sampling can be expressed exactly as public `kumo.pytorch` recent sampling. +Uniform sampling into timeless tables and empty graph layers fail explicitly. +The old `local_native` and `temporal_query` selection implementations are +comparison references, not fallbacks in this package. A supplied index must +match the dataset snapshot, contract fingerprint, and recorded dependency +provenance; an unrelated or incomplete index is refused. + +| Contract area | Status | +| --- | --- | +| One or both frozen context/predict splits; ordered instance selection | Supported | +| Classification and regression targets; context label availability checks | Supported | +| Temporal recent sampling, forward/reverse steps, multiple relations and hops | Supported through the public columnar graph API | +| Per-instance identity, selected edges, nulls, empty related tables, exact int64 IDs | Preserved in typed Arrow tables and both encodings | +| Bundle hashes, source schema, keys/FKs, clock precision, rows/columns/bytes | Checked before or during sampling/encoding | +| Uniform sampling into timeless destinations, empty graph layers | Explicitly rejected | +| `local_native` and `temporal_query` selection | Comparison-only; no production fallback | + +| Prior entry point | This package | +| --- | --- | +| `Bundle(root)`, `Workload(bundle, spec)` | `fm.prepare(root, spec, index=...)` | +| `KumoPytorchBackend.prepare_batch(seeds)` | `loader.materialize(context=..., predict=...)` | +| `build_arrow_tables(backend, workload)` | `request.load().as_arrow_request()` | +| `build_payload(..., encoding="arrays")` | `batch.encode("arrays")` | +| `build_payload(..., encoding="arrow_ipc")` | `batch.encode("arrow_ipc")` | +| `arrow_to_sdm(request)` | `examples/local_sdm/adapter.py:arrow_to_sdm(batch)` | + +Source baselines: fm-dataloader `b4516394e98929c5b4752745866d5e3138276255`; +engine `9637e9cc9e2da877959b3c6b9db88c0fc7e35f3f`. The public columnar +API landed in engine PRs #340 and #365. diff --git a/packages/experimental/kumo-fm/examples/local_sdm/README.md b/packages/experimental/kumo-fm/examples/local_sdm/README.md new file mode 100644 index 000000000..246fd8767 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/README.md @@ -0,0 +1,48 @@ +# Optional local SDM example + +`connector_to_prediction.py` is the primary connector-to-model example. Run it +from this directory after installing matching `kumo-engine`, DiskGraph, +`kumo-pytorch` and `kumo-fm` wheels from one engine revision, plus the pinned +SDM checkout in `requirements.lock`. `kumo.engine.datasets.shop()` supplies +the reproducible 1,000-user shop Parquet sample; the script writes producer and +consumer state under `scratch/connector-to-prediction/` here. The public +`KumoRelational` classifier downloads its pretrained checkpoint on first use; +set `HF_HOME` to a writable model-cache directory. A CUDA device is used when +available, otherwise CPU. The example's 30-day future label window ends well +before the November query cutoff. Its one sampling hop is independent of the +model's two-hop setting. No SDM package is imported by `kumo.fm` itself. + +```bash +cd packages/experimental/kumo-fm/examples/local_sdm +python connector_to_prediction.py +``` + +The separate `walkthrough.ipynb` uses the included frozen bundle fixture and +keeps SDM and its Arrow compatibility handling beside the notebook. Install +matching DiskGraph, `kumo-pytorch`, and `kumo-fm` wheels +from one engine revision, then the pinned SDM environment in +`requirements.lock`. The model may download the +`nvidia/Kumo-Relational` v1.0.1 checkpoint; the prior validation used +snapshot `2bd603d3d8f25f67a7aaa8579908595e20567e22`. + +The notebook includes `fixture/bundle/` and `fixture/workload.json`. Run it +from this directory to use them by default. Set `FM_SDM_FIXTURE` to +another directory containing `bundle/` and `workload.json` to use your own +eligible `native_multihop` workload. Set `FM_SDM_BUNDLE` when its bundle has +another path. The notebook opens or creates `fm-index/` in the fixture directory, samples one +batch, converts its retained Arrow tables through public +`TableTensor.from_arrow` and `RelatedTables`, and exercises both model +call forms. The helper only repairs zero-chunk columns and read-only string +buffers when SDM requires it. The same batch is encoded as arrays and IPC. + +Regenerate the included smoke case with +`python benchmark/fm-payloads/make_fixture.py /tmp/fm-case` from the repository +root, then copy `bundle/` and `workload.json` into this directory's `fixture/`. +You can also set `FM_SDM_FIXTURE=/tmp/fm-case` to use the generated copy. +The notebook includes the local execution output: both model call forms +agreed with a maximum absolute difference of 0.0 on this fixture. + +Direct conversion avoids HTTP serialization; it does not promise that SDM +tensor construction or inference is zero copy. The source baseline is +fm-dataloader PR #27 at +`b4516394e98929c5b4752745866d5e3138276255`. diff --git a/packages/experimental/kumo-fm/examples/local_sdm/adapter.py b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py new file mode 100644 index 000000000..aee1efc6a --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Example-local conversion from a trusted FM Arrow batch to public SDM inputs.""" +import pyarrow as pa + + +def _compatible_buffers(table): + for index, field in enumerate(table.schema): + column = table.column(index) + if column.num_chunks == 0: + table = table.set_column(index, field, pa.array([], type=field.type)) + elif pa.types.is_string(field.type) or pa.types.is_large_string(field.type): + if any(buffer is not None and not buffer.is_mutable + for chunk in column.chunks for buffer in chunk.buffers()): + table = table.set_column(index, field, pa.concat_arrays(column.chunks)) + return table + + +def arrow_to_sdm(batch): + """Return the one-shot KumoRelational arguments from one retained batch.""" + from sdm import RelatedTables, TableTensor + + payload = batch.as_arrow_request() + if not {"context", "predict"} <= payload.keys(): + raise ValueError("SDM prediction needs both context and predict splits") + schema = payload["schema"] + target = payload["task"]["target"]["column_name"] + target_stype = "numerical" if payload["task"]["kind"] == "regression" else "categorical" + relationships, links = [], [] + for relation in schema["relationships"]: + if relation.get("source_table", "instance_table") == "instance_table": + links.append(dict(task_columns=relation["source_columns"], + table=relation["target_table"], + table_columns=relation["target_columns"])) + else: + relationships.append(dict(left_table=relation["source_table"], + left_columns=relation["source_columns"], + right_table=relation["target_table"], + right_columns=relation["target_columns"])) + instances, related = {}, {} + for split in ("context", "predict"): + tables = {} + entries = {"instance_table": payload[split]["instance_table"], + **payload[split]["related_tables"]} + for name, table in entries.items(): + spec = schema["instance_table"] if name == "instance_table" else schema["related_tables"][name] + stypes = {} + for column in table.column_names: + field = spec["columns"][column] + stype = field.get("stype") + if name == "instance_table" and column == target: + stype = target_stype + elif stype is None: + stype = {"bool": "categorical", "string": "categorical", + "timestamp[us]": "datetime", "date": "datetime"}.get( + field["dtype"], "numerical") + stypes[column] = {"timestamp": "datetime", "id": "id"}.get( + stype.lower(), stype.lower()) + tables[name] = TableTensor.from_arrow( + _compatible_buffers(table), stypes=stypes, device="cpu") + instances[split] = tables.pop("instance_table") + related[split] = RelatedTables( + tables=tables, relationships=relationships, task_links=links) + if target in instances["predict"].stypes: + raise ValueError("Prediction inputs must not contain the target") + return dict(x_context=instances["context"].drop_columns(target), + y_context=instances["context"][[target]], + x_query=instances["predict"], + related_context_tables=related["context"], + related_query_tables=related["predict"]) diff --git a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py new file mode 100644 index 000000000..242c9213c --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Connector → snapshot → retained FM Arrow batch → local SDM prediction. + +Run from this directory after installing the matching engine/native wheels and +the optional SDM environment. See README.md for the checkpoint and dataset. +""" +from datetime import datetime, timezone +from pathlib import Path + +import torch +from adapter import arrow_to_sdm +from kumoapi.typing import Stype +from sdm.models import KumoRelational + +import kumo.engine as kumo +from kumo import fm +from kumo.engine import datasets +from kumo.pytorch import api + +root = Path("scratch/connector-to-prediction").resolve() +root.mkdir(parents=True, exist_ok=True) +kumo.configure(scratch_dir=root / "producer") +source = kumo.FileConnector(root_dir=datasets.shop()) +graph = kumo.Graph( + tables={ + "users": kumo.Table.from_source_table( + source["users.parquet"], primary_key="user_id"), + "orders": kumo.Table.from_source_table( + source["orders.parquet"], primary_key="order_id", + time_column="order_time").with_column("user_id", stype=Stype.ID), + }, + edges=[kumo.Edge(src_table="orders", fkey="user_id", dst_table="users")], +) +engine = api.DiskGraphEngine.from_snapshot( + graph.snapshot(), work_dir=root / "consumer", num_threads=2) +relation = engine.schema.relations[0].name + +horizon = 30 * 24 * 60 * 60 +sample = api.SampleSpec( + entity=api.EntitySpec("users"), + targets={"next_orders": api.Compare( + api.Aggregate(api.Rows(path=(relation,), time=api.TimeWindow( + "order_time", start=0, stop=horizon)), "count"), + "greater_than", 0)}, + neighborhoods={"history": api.NeighborhoodSpec(layers=( + api.GraphLayer(samples=(api.GraphSample( + relation=relation, direction="reverse", strategy="recent", fanout=8),)), + ))}, +) + +with fm.prepare(engine=engine, sample=sample, target="next_orders", + task="binary_classification", classes={False: "no", True: "yes"}, + features={"users": ["user_id"], + "orders": ["order_id", "order_time"]}) as loader: + batch = loader.load( + context=fm.Context(entities=range(32), times=[ + datetime(2025, 6, 1, tzinfo=timezone.utc)]), + query=fm.Query(entities=range(8), + at=datetime(2025, 11, 1, tzinfo=timezone.utc)), + ) + +device = "cuda" if torch.cuda.is_available() else "cpu" +inputs = {name: value.to(device) for name, value in arrow_to_sdm(batch).items()} +model = KumoRelational(task="classification", device=device).eval() +with torch.inference_mode(): + model.fit(inputs["x_context"], inputs["y_context"], + inputs["related_context_tables"], num_estimators=1, num_hops=2, + generator=torch.Generator(device=device).manual_seed(42)) + predictions = model.predict(inputs["x_query"], inputs["related_query_tables"]) +print(predictions.to_arrow()) +print("Same retained batch, HTTP bytes:", len(batch.encode("arrays")), + len(batch.encode("arrow_ipc"))) diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json new file mode 100644 index 000000000..81c44a755 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json @@ -0,0 +1,4 @@ +{ + "manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", + "validation_sha256": "359d774e6cbd42a0b3652022842dc2e228e8488527b6a9ee3e87352dc7a40003" +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json new file mode 100644 index 000000000..ad24cc00c --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json @@ -0,0 +1,223 @@ +{ + "format_version": 2, + "logical_id": "b1b964d2a964e4fb95b4cb093d5de467199f2b10b67970e8fe3b5407d7b8137a", + "recipe": { + "generator": "kumo_fm_integration" + }, + "tables": { + "users": { + "primary_key": "user_id", + "columns": { + "user_id": { + "dtype": "int64", + "nullable": false + }, + "segment": { + "dtype": "string", + "nullable": false, + "stype": "categorical" + }, + "active": { + "dtype": "bool", + "nullable": false + } + }, + "time_column": null, + "availability": "timeless", + "rows": 3, + "files": [ + "tables/users/part-00000.parquet" + ] + }, + "interactions": { + "primary_key": "interaction_id", + "columns": { + "interaction_id": { + "dtype": "int64", + "nullable": false + }, + "user_id": { + "dtype": "int64", + "nullable": true + }, + "item_id": { + "dtype": "string", + "nullable": true + }, + "event_time": { + "dtype": "timestamp[us]", + "nullable": true + }, + "amount": { + "dtype": "float64", + "nullable": true, + "stype": "numerical" + } + }, + "time_column": "event_time", + "rows": 4, + "files": [ + "tables/interactions/part-00000.parquet" + ] + }, + "items": { + "primary_key": "item_id", + "columns": { + "item_id": { + "dtype": "string", + "nullable": false + }, + "category_id": { + "dtype": "int64", + "nullable": false + }, + "description": { + "dtype": "string", + "nullable": false, + "stype": "text" + } + }, + "time_column": null, + "availability": "timeless", + "rows": 3, + "files": [ + "tables/items/part-00000.parquet" + ] + }, + "categories": { + "primary_key": "category_id", + "columns": { + "category_id": { + "dtype": "int64", + "nullable": false + }, + "name": { + "dtype": "string", + "nullable": false, + "stype": "categorical" + } + }, + "time_column": null, + "availability": "timeless", + "rows": 2, + "files": [ + "tables/categories/part-00000.parquet" + ] + } + }, + "relations": { + "interaction_user": { + "child": "interactions", + "foreign_key": "user_id", + "parent": "users" + }, + "interaction_item": { + "child": "interactions", + "foreign_key": "item_id", + "parent": "items" + }, + "item_category": { + "child": "items", + "foreign_key": "category_id", + "parent": "categories" + } + }, + "tasks": { + "binary_classification": { + "entity_table": "users", + "kind": "binary_classification", + "target": { + "dtype": "string", + "classes": [ + "no", + "yes" + ] + }, + "label_horizon_us": 2592000000000, + "context": "tasks/binary_classification/context.parquet", + "predict": "tasks/binary_classification/predict.parquet", + "labels": "tasks/binary_classification/labels.parquet" + }, + "regression": { + "entity_table": "users", + "kind": "regression", + "target": { + "dtype": "int64" + }, + "label_horizon_us": 2592000000000, + "context": "tasks/regression/context.parquet", + "predict": "tasks/regression/predict.parquet", + "labels": "tasks/regression/labels.parquet" + } + }, + "files": { + "tables/users/part-00000.parquet": { + "sha256": "3bbcc77313b1cfc069b64512b7b1ff84b006dea5785df1822dc7e839bc522154", + "rows": 3, + "bytes": 913, + "row_groups": [ + 3 + ] + }, + "tables/interactions/part-00000.parquet": { + "sha256": "0e7fed483ac455b984f28b46438ef4d4de8549f30c8c8eb134cf8f24a8baffb5", + "rows": 4, + "bytes": 2730, + "row_groups": [ + 3, + 1 + ] + }, + "tables/items/part-00000.parquet": { + "sha256": "baab8536005f41365a3b347c1051ebae1437665d399600a758d5497eb2ed0474", + "rows": 3, + "bytes": 1056, + "row_groups": [ + 3 + ] + }, + "tables/categories/part-00000.parquet": { + "sha256": "b8f5211b06fd6cb5cba9b4c5680cadfa775920b571ea21f6fd36d8b37cd3faf8", + "rows": 2, + "bytes": 755, + "row_groups": [ + 2 + ] + }, + "tasks/binary_classification/context.parquet": { + "sha256": "5e13ecd914bca54712a062c191ffaff326998d1b93e03a11ce66753d06e4603d", + "rows": 2, + "bytes": 1689 + }, + "tasks/binary_classification/predict.parquet": { + "sha256": "2fcf0a7d39150dc136caf5ddf06c51156eb0e147dfa50c6914141eb78f3dc256", + "rows": 4, + "bytes": 1148 + }, + "tasks/binary_classification/labels.parquet": { + "sha256": "6cb584811b24e6267262be6b26c8c9236a61181e5b90766f92e2da104705dad5", + "rows": 4, + "bytes": 748 + }, + "tasks/regression/context.parquet": { + "sha256": "df3cf924ddc362b94ea6a7672ebb3e4ed52838a42bcf7b04e5520a91e939c19b", + "rows": 2, + "bytes": 1750 + }, + "tasks/regression/predict.parquet": { + "sha256": "2fcf0a7d39150dc136caf5ddf06c51156eb0e147dfa50c6914141eb78f3dc256", + "rows": 4, + "bytes": 1148 + }, + "tasks/regression/labels.parquet": { + "sha256": "204b2a6f889f3a92e9db52b54d54e14ff6ad043be2a05ab26f57e1a68329ef6a", + "rows": 4, + "bytes": 808 + } + }, + "writer": { + "pyarrow": "24.0.0", + "row_group_size": 3, + "reverse_layout": false + } +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet new file mode 100644 index 000000000..7436497b8 Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/interactions/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/interactions/part-00000.parquet new file mode 100644 index 000000000..dad3006da Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/interactions/part-00000.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet new file mode 100644 index 000000000..002ecc0da Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet new file mode 100644 index 000000000..2d39202d9 Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet new file mode 100644 index 000000000..0873540bc Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet new file mode 100644 index 000000000..49d0f2826 Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet new file mode 100644 index 000000000..dcbf9b7ed Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet new file mode 100644 index 000000000..cc7046d09 Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet new file mode 100644 index 000000000..21c27b95d Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet new file mode 100644 index 000000000..dcbf9b7ed Binary files /dev/null and b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet differ diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json new file mode 100644 index 000000000..69edb8031 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json @@ -0,0 +1,31 @@ +{ + "status": "passed", + "tables": { + "users": { + "rows": 3 + }, + "interactions": { + "rows": 4 + }, + "items": { + "rows": 3 + }, + "categories": { + "rows": 2 + } + }, + "relations": { + "interaction_user": { + "null_links": 0, + "edges": 4 + }, + "interaction_item": { + "null_links": 0, + "edges": 4 + }, + "item_category": { + "null_links": 0, + "edges": 3 + } + } +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json new file mode 100644 index 000000000..7cb94b0e1 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "logical_id": "b1b964d2a964e4fb95b4cb093d5de467199f2b10b67970e8fe3b5407d7b8137a", + "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", + "task": "binary_classification", + "prediction_cutoff": "2026-02-01T00:00:00Z", + "selection": "native_multihop", + "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", + "layers": [ + [ + { + "relation": "interaction_user", + "direction": "reverse", + "fanout": 1 + } + ] + ], + "projections": { + "users": [ + "user_id", + "segment", + "active" + ], + "interactions": [ + "interaction_id", + "user_id", + "item_id", + "event_time", + "amount" + ] + }, + "encoding": "arrays" +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock b/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock new file mode 100644 index 000000000..8be1ce059 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock @@ -0,0 +1,3 @@ +pyarrow==24.0.0 +torch==2.13.0 +structured-data-models @ git+https://github.com/NVIDIA/structured-data-models.git@fd2c29d035a165a998ea4fdb2b08a81fd89f71e1 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb new file mode 100644 index 000000000..d85a84a66 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fm-overview", + "metadata": {}, + "source": [ + "# Local FM Arrow to SDM\n", + "\n", + "Run from this directory to use the included validated fixture, or set `FM_SDM_FIXTURE` for your own bundle and workload. Install the matching DiskGraph and pinned SDM environment described in this folder's README. This notebook samples once and reuses the same Arrow batch for local inference and both HTTP bodies.\n" + ] + }, + { + "cell_type": "code", + "id": "fm-load", + "metadata": {}, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "instance_id: int64 not null\n", + "__fm_entity_key: int64 not null\n", + "__fm_anchor: timestamp[us, tz=UTC] not null\n", + "__fm_target: string not null\n", + "584\n" + ] + } + ], + "source": [ + "import os\n", + "from pathlib import Path\n", + "import torch\n", + "from kumo import fm\n", + "from adapter import arrow_to_sdm\n", + "from sdm.models import KumoRelational\n", + "\n", + "fixture = Path(os.environ.get(\"FM_SDM_FIXTURE\", \"fixture\"))\n", + "torch.set_num_threads(2)\n", + "bundle_path = Path(os.environ.get(\"FM_SDM_BUNDLE\", str(fixture / \"bundle\")))\n", + "with fm.prepare(bundle_path, fixture / \"workload.json\",\n", + " index=fixture / \"fm-index\", threads=2) as loader:\n", + " with loader.materialize() as request:\n", + " batch = request.load()\n", + "print(batch.tables(\"context\")[\"instance_table\"].schema)\n", + "print(batch.logical_arrow_bytes)\n" + ] + }, + { + "cell_type": "code", + "id": "fm-convert", + "metadata": {}, + "execution_count": 2, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'arrays_bytes': 3374, 'ipc_bytes': 8859}\n" + ] + } + ], + "source": [ + "inputs = arrow_to_sdm(batch)\n", + "arrays_body = batch.encode(\"arrays\")\n", + "ipc_body = batch.encode(\"arrow_ipc\")\n", + "print({\"arrays_bytes\": len(arrays_body), \"ipc_bytes\": len(ipc_body)})\n" + ] + }, + { + "cell_type": "code", + "id": "fm-infer", + "metadata": {}, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "yes: float\n", + "one-shot/fit-predict max absolute difference: 0.0\n" + ] + } + ], + "source": [ + "model = KumoRelational(task=\"classification\", device=\"cpu\")\n", + "with torch.inference_mode():\n", + " one_shot = model(**inputs, num_estimators=1,\n", + " generator=torch.Generator().manual_seed(42), num_hops=2)\n", + " model.fit(inputs[\"x_context\"], inputs[\"y_context\"],\n", + " inputs[\"related_context_tables\"], num_estimators=1,\n", + " generator=torch.Generator().manual_seed(42), num_hops=2)\n", + " reused_context = model.predict(inputs[\"x_query\"], inputs[\"related_query_tables\"])\n", + "torch.testing.assert_close(one_shot.numerical, reused_context.numerical,\n", + " rtol=1e-5, atol=1e-5)\n", + "print(one_shot.to_arrow().schema)\n", + "print(\"one-shot/fit-predict max absolute difference:\",\n", + " float((one_shot.numerical - reused_context.numerical).abs().max()))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "fm_validation": { + "result": "passed", + "sdm_revision": "fd2c29d035a165a998ea4fdb2b08a81fd89f71e1", + "model_snapshot": "2bd603d3d8f25f67a7aaa8579908595e20567e22", + "wheel_sha256": "c55014ca6868058f9ddbd4efdcc57032eba24e9152dc445e6e4d95e56edc5818", + "fixture_snapshot": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/experimental/kumo-fm/pyproject.toml b/packages/experimental/kumo-fm/pyproject.toml new file mode 100644 index 000000000..dc7d40562 --- /dev/null +++ b/packages/experimental/kumo-fm/pyproject.toml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["setuptools>=77", "pyarrow==24.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "kumo-fm" +version = "0.1.0a1" +description = "FM payload loader over Kumo PyTorch and DiskGraph" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE"] +dependencies = [ + "duckdb==2.0.0", + "kumo-pytorch>=0.1.0a1", + "numpy>=1.26,<3", + "pyarrow==24.0.0", +] + +[project.optional-dependencies] +test = ["pytest>=8,<10"] +relbench = ["PyYAML>=6,<7"] + +[tool.uv.sources] +kumo-pytorch = {workspace = true} + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] +include = ["kumo.fm*"] +namespaces = true + +[tool.setuptools.package-data] +"kumo.fm._dataset_support" = ["presets/*.json"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/packages/experimental/kumo-fm/setup.py b/packages/experimental/kumo-fm/setup.py new file mode 100644 index 000000000..ccc6cdbbe --- /dev/null +++ b/packages/experimental/kumo-fm/setup.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pyarrow +from setuptools import Extension, setup + +setup(ext_modules=[Extension( + "kumo.fm._native_json", ["src/kumo/fm/_native_json.cpp"], + include_dirs=[pyarrow.get_include()], language="c++", + extra_compile_args=["-std=c++17", "-O3"], +)]) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py new file mode 100644 index 000000000..0a1a60725 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FM payload assembly through public Kumo PyTorch columnar batches.""" +from .batch import PayloadBatch +from .loader import FMLoader, PreparedRequest, prepare +from ._snapshot import Context, Query + +__all__ = ["Context", "Query", "FMLoader", "PreparedRequest", "PayloadBatch", "prepare"] diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py new file mode 100644 index 000000000..14e680031 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py @@ -0,0 +1,149 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate and assemble selected columnar graph occurrences into FM tables.""" +from collections import defaultdict + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + +from ._phase import phase +from .bundle import TYPES + + +def empty_features(columns): + return pa.table({name: pa.array([], type=TYPES[spec["dtype"]]) for name, spec in columns.items()}) + + +def normalize_features(table, columns): + return pa.table({name: pc.cast(table[name], TYPES[columns[name]["dtype"]]) for name in table.column_names}) + + +def indices(column): + return column.to_numpy(zero_copy_only=False) + + +def _parent_positions(pairs, child_count, relation): + """Canonicalize validated local endpoints without sorting edge pairs. + + A child has at most one selected parent per relation. Scatter all batches + first, then compare *every* original pair with the final map: this catches + conflicts both within and across batches regardless of which repeated + assignment NumPy retains. Local endpoint bounds/ownership are checked by + the caller; -1 denotes an unselected child, not a source primary key. + """ + with phase('native_multihop_parent_map_allocate', 'graph_assembly', relation=relation): + positions = np.full(child_count, -1, dtype=np.int64) + with phase('native_multihop_parent_map_scatter', 'graph_assembly', relation=relation): + for child, parent in pairs: + positions[child] = parent + with phase('native_multihop_parent_map_validate', 'graph_assembly', relation=relation): + for child, parent in pairs: + if not np.array_equal(positions[child], parent): + raise ValueError('Selected child has conflicting parents') + with phase('native_multihop_parent_map_count', 'graph_assembly', relation=relation): + count = int(np.count_nonzero(positions >= 0)) + return positions, count + + +class GraphAssembler: + def __init__(self, bundle): + self.bundle = bundle + self.node_types = list(bundle.tables) + + def _adapt(self, workload, seeds, ids, nodes, features, edge_batches, directions): + """Build final related tables using columnar gathers, not Python edges.""" + from ._contract import schema_for + schema = schema_for(workload)['related_tables'] + frames, owners, node_tables, orders = {}, {}, {}, {} + for name in workload.table_names: + frames[name] = empty_features(self.bundle.tables[name]['columns']) + owners[name] = np.array([], dtype=np.int64) + for type_id in set(nodes) | set(features): + if not nodes[type_id] or not features[type_id]: + raise ValueError('Native fused node/feature alignment mismatch') + name = self.node_types[type_id] + with phase('native_multihop_feature_normalize', 'graph_assembly', table=name): + ns = pa.Table.from_batches(nodes[type_id]) + fs = normalize_features(pa.Table.from_batches(features[type_id]), self.bundle.tables[name]['columns']) + with phase('native_multihop_node_validate', 'graph_assembly', table=name): + if (len(ns) != len(fs) or ns['id_a'].type != pa.uint64() or ns['id_b'].type != pa.uint64() + or ns['id_a'].null_count or ns['id_b'].null_count): + raise ValueError('Native fused node/feature alignment mismatch') + owner = indices(ns['id_a']) + if np.any(owner >= len(seeds)) or np.any(owner < 0): + raise ValueError('Native seed position out of bounds') + if len(fs) > workload.budgets['max_rows_per_table']: + raise ValueError('Expanded related table exceeds row budget') + with phase('native_multihop_identity_sort', 'graph_assembly', table=name): + pk = fs[self.bundle.tables[name]['primary_key']] + identity = pa.table({'i': owner, 'k': pk}) + order = pc.sort_indices(identity, sort_keys=[('i', 'ascending'), ('k', 'ascending')]) + with phase('native_multihop_identity_validate', 'graph_assembly', table=name): + ordered = identity.take(order) + duplicate = pc.and_(pc.equal(ordered['i'].slice(1), ordered['i'].slice(0, max(0, len(fs)-1))), + pc.equal(ordered['k'].slice(1), ordered['k'].slice(0, max(0, len(fs)-1)))) + if pk.null_count or pc.any(duplicate).as_py(): + raise ValueError('Native returned duplicate or null node occurrences') + frames[name], owners[name], node_tables[name], orders[name] = fs, owner, ns, order + with phase('native_multihop_root_validate', 'graph_assembly'): + root = frames[workload.entity] + root_spec = self.bundle.tables[workload.entity] + root_keys = root[root_spec['primary_key']].slice(0, len(seeds)) + if (len(root_keys) != len(seeds) + or not np.array_equal(owners[workload.entity][:len(seeds)], np.arange(len(seeds))) + or not pc.all(pc.fill_null(pc.equal(root_keys, pa.array( + [s['entity_key'] for s in seeds], type=root_keys.type)), False)).as_py()): + raise ValueError('Native root prefix differs from input seeds') + if not np.array_equal(indices(node_tables[workload.entity]['id_b'].slice(0, len(seeds))), np.asarray(ids)): + raise ValueError('Native root IDs differ from resolved seeds') + if root_spec.get('time_column'): + from .bundle import TYPES + anchors = pa.array([s['anchor_time'] for s in seeds], TYPES['timestamp[us]']) + if not pc.all(pc.fill_null(pc.less_equal(root[root_spec['time_column']].slice(0, len(seeds)), anchors), False)).as_py(): + raise ValueError('Task entity is not eligible at its anchor') + + # Positions stay in the engine's original node order until final take(). + pairs = defaultdict(list) + for rel_id, batches in edge_batches.items(): + relation, direction, source, dest = directions[rel_id] + with phase('native_multihop_edge_validate', 'graph_assembly', relation=relation, direction=direction): + batch = pa.Table.from_batches(batches) + if (batch['id_a'].type != pa.uint64() or batch['id_b'].type != pa.uint64() + or batch['id_a'].null_count or batch['id_b'].null_count): + raise ValueError('Native edge has null endpoints') + aa, bb = indices(batch['id_a']), indices(batch['id_b']) + if np.any(aa >= len(frames[source])) or np.any(bb >= len(frames[dest])): + raise ValueError('Native edge position out of bounds') + if not np.array_equal(owners[source][aa], owners[dest][bb]): + raise ValueError('Native edge crosses seed instances') + child, parent = (aa, bb) if direction == 'forward' else (bb, aa) + pairs[relation].append((child, parent)) + edge_columns, edge_count = {}, 0 + for relation in workload.relation_names: + rel = self.bundle.relations[relation] + parent = frames[rel['parent']][self.bundle.tables[rel['parent']]['primary_key']] + positions, count = _parent_positions(pairs[relation], len(frames[rel['child']]), relation) + edge_count += count + with phase('native_multihop_parent_key_gather', 'graph_assembly', relation=relation): + edge_columns[relation] = pc.take(parent, pa.array(positions, mask=positions < 0)) + with phase('native_multihop_instance_ids', 'graph_assembly'): + instance_ids = pa.array([s.get('instance_id', i) for i, s in enumerate(seeds)], pa.int64()) + ready = {} + for name, spec in schema.items(): + columns = {} + for col in spec['columns']: + if col == 'instance_id': + with phase('native_multihop_instance_gather', 'graph_assembly', table=name): + columns[col] = pc.take(instance_ids, pa.array(owners[name])) + elif col.startswith('__fm_edge_'): + columns[col] = edge_columns[col.removeprefix('__fm_edge_')] + else: + columns[col] = frames[name][col] + with phase('native_multihop_table_build', 'graph_assembly', table=name): + table = pa.table(columns) + with phase('native_multihop_output_gather', 'graph_assembly', table=name): + ready[name] = table.take(orders[name]) if name in orders else table + return ready diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py new file mode 100644 index 000000000..ba0273a8a --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FM schema and task metadata shared by direct and wire output.""" + + +def schema_for(workload): + bundle = workload.bundle + id_spec = {"dtype": "int64", "stype": "ID", "nullable": False} + entity = bundle.tables[workload.entity] + key_spec = {"dtype": entity["columns"][entity["primary_key"]]["dtype"], "stype": "ID", "nullable": False} + instance = {"primary_key": "instance_id", "columns": { + "instance_id": id_spec, "__fm_entity_key": key_spec, + "__fm_anchor": {"dtype": "timestamp[us]", "stype": "timestamp", "nullable": False}, + "__fm_target": {"dtype": workload.target["dtype"] if workload.kind == "regression" else "string", "nullable": False}}} + related = {} + for name in sorted(workload.table_names): + table = bundle.tables[name] + pk = table["primary_key"] + columns = {"instance_id": id_spec, + pk: {"dtype": table["columns"][pk]["dtype"], "stype": "ID", "nullable": False}} + fks = {r["foreign_key"] for r in bundle.relations.values() if r["child"] == name} + for col in workload.projections[name]: + spec = dict(table["columns"][col]) + if spec["dtype"] == "date": + spec["dtype"] = "timestamp[us]" + if col == pk or col in fks: + spec["stype"] = "ID" + columns[col] = spec + related[name] = {"primary_key": ["instance_id", pk], "columns": columns} + relationships = [{"source_columns": ["instance_id", "__fm_entity_key"], + "target_table": workload.entity, + "target_columns": ["instance_id", entity["primary_key"]]}] + for name in sorted(workload.relation_names): + rel = bundle.relations[name] + parent = bundle.tables[rel["parent"]] + dtype = parent["columns"][parent["primary_key"]]["dtype"] + col = "__fm_edge_" + name + related[rel["child"]]["columns"][col] = {"dtype": dtype, "stype": "ID", "nullable": True} + relationships.append({"source_table": rel["child"], "source_columns": ["instance_id", col], + "target_table": rel["parent"], "target_columns": ["instance_id", parent["primary_key"]]}) + return {"instance_table": instance, "related_tables": related, "relationships": relationships} + + +def metadata_for(workload): + target = dict(workload.target) + if workload.kind != "regression": + target.update(dtype="string", classes=[str(value) for value in target["classes"]]) + fields = ["prediction"] if workload.kind == "regression" else ["prediction", "probabilities"] + return { + "model": "kumo-relational", + "task": { + "kind": workload.kind, + "entity_table_names": [workload.entity], + "anchor_time_column": "__fm_anchor", + "target": {"table_name": "instance_table", "column_name": "__fm_target", **target}, + }, + "schema": schema_for(workload), + "output": {"fields": fields, "score_format": "object"}, + } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py new file mode 100644 index 000000000..317e46a9a --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Portable, deterministic synthetic relational Parquet bundles.""" + +__version__ = "1.1.0" diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py new file mode 100644 index 000000000..0c17003d2 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py @@ -0,0 +1,284 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve small declarative presets; reject unsupported knobs before writing.""" + +import json +import math +import re +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from importlib.resources import files + +from . import __version__ + +ALIASES = dict( + zip( + [ + "v_marketo_lead", + "engagements_v2", + "marketing_kumo_bronze_rpt_les_metrics_marketo_agg", + "marketing_kumo_bronze_rpt_les_metrics_marketo_usecase_agg", + "marketing_kumo_bronze_v_marketo_u2_conversions", + "marketing_kumo_bronze_v_test_marketo_trigger_activities", + "v_qual_notes", + ], + [ + "leads", + "engagements", + "metrics", + "usecase_metrics", + "conversions", + "activities", + "notes", + ], + ) +) + + +@dataclass +class Config: + preset: str = "commerce_v1" + scale: float = 0.001 + seed: int = 42 + rows: dict = field(default_factory=dict) + # Overrides keyed by table.column; omitted Marketo relations retain legacy ratios. + relations: dict = field(default_factory=dict) + numeric_features: int = 0 + categorical_features: int = 0 + text_features: int = 0 + category_cardinality: int = 32 + text_length: int = 64 + feature_null_fraction: float = 0.0 + history_start: str = "2023-01-01" + history_days: int = 1095 + recent_share: float = 0.0 + batch_rows: int = 65536 + file_rows: int = 1000000 + row_group_rows: int = 100000 + context_rows: int = 64 + prediction_rows: int = 256 # per degree cohort + lookback_days: int = 365 + threads: int = 2 + memory_limit: str = "2GB" + max_output_gib: float = 8.0 + min_free_gib: float = 5.0 + + def resolve(self): + if self.preset not in ("marketo_v1", "commerce_v1"): + raise ValueError("preset must be marketo_v1 or commerce_v1") + if not math.isfinite(self.scale) or self.scale <= 0: + raise ValueError("scale must be finite and positive") + if not isinstance(self.seed, int) or not 0 <= self.seed < 2**64: + raise ValueError("seed must be an unsigned 64-bit integer") + for name in ( + "batch_rows", + "file_rows", + "row_group_rows", + "context_rows", + "prediction_rows", + "category_cardinality", + "text_length", + "history_days", + "lookback_days", + "threads", + ): + if not isinstance(getattr(self, name), int) or getattr(self, name) < 1: + raise ValueError(f"{name} must be a positive integer") + if self.history_days < 120: + raise ValueError( + "history_days must be at least 120 for complete context/predict labels" + ) + for name in ("numeric_features", "categorical_features", "text_features"): + if not isinstance(getattr(self, name), int) or getattr(self, name) < 0: + raise ValueError(f"{name} must be a nonnegative integer") + for name in ("feature_null_fraction", "recent_share"): + if ( + not math.isfinite(getattr(self, name)) + or not 0 <= getattr(self, name) <= 1 + ): + raise ValueError(f"{name} must be in [0, 1]") + for name in ("max_output_gib", "min_free_gib"): + if not math.isfinite(getattr(self, name)) or getattr(self, name) < 0: + raise ValueError(f"{name} must be finite and nonnegative") + if self.max_output_gib == 0: + raise ValueError("max_output_gib must be positive") + if not re.fullmatch(r"[1-9][0-9]*(MB|GB)", self.memory_limit): + raise ValueError("memory_limit must be an integer number of MB or GB") + start = datetime.strptime(self.history_start, "%Y-%m-%d").replace(tzinfo=UTC) + tables = marketo_tables() if self.preset == "marketo_v1" else commerce_tables() + if set(self.rows) - set(tables): + raise ValueError( + f"Unknown row-count tables: {set(self.rows) - set(tables)}" + ) + for name, table in tables.items(): + n = self.rows.get(name, max(1, round(table["base_rows"] * self.scale))) + if ( + not isinstance(n, int) + or n < (0 if table["time_column"] else 1) + or n > 2**63 - 1 + ): + raise ValueError(f"Invalid row count for {name}: {n}") + table["rows"] = n + for kind, count, dtype in [ + ("numeric", self.numeric_features, "float64"), + ("category", self.categorical_features, "string"), + ("text", self.text_features, "string"), + ]: + for i in range(count): + table["columns"].append( + {"name": f"extra_{kind}_{i}", "type": dtype, "role": kind} + ) + known = set() + for name, table in tables.items(): + for rel in table["fkeys"]: + key = f"{name}.{rel['column']}" + known.add(key) + override = self.relations.get(key, {}) + unknown = set(override) - { + "active_fraction", + "null_fraction", + "distribution", + "hot_fraction", + "hot_share", + "power", + } + if unknown: + raise ValueError(f"Unknown relation options for {key}: {unknown}") + rel.update(override) + for k in ( + "active_fraction", + "null_fraction", + "hot_fraction", + "hot_share", + ): + if not math.isfinite(rel[k]) or not 0 <= rel[k] <= 1: + raise ValueError(f"{key}.{k} must be in [0, 1]") + if rel["distribution"] not in ("uniform", "hotspot", "power"): + raise ValueError(f"Unknown distribution for {key}") + if not math.isfinite(rel["power"]) or rel["power"] <= 0: + raise ValueError("power must be finite and positive") + parents = tables[rel["table"]]["rows"] + linked = round(table["rows"] * (1 - rel["null_fraction"])) + active = ( + min(parents, max(1, round(parents * rel["active_fraction"]))) + if linked + else 0 + ) + # At tiny scales the desired source ratios can be infeasible. Record actual coverage. + active = min(active, linked) + if linked and rel["active_fraction"] == 0: + raise ValueError(f"{key}: linked rows require active parents") + rel.update(linked_rows=linked, active_parents=active) + if set(self.relations) - known: + raise ValueError(f"Unknown relations: {set(self.relations) - known}") + return { + "generator_version": __version__, + "timestamp_granularity": "seconds", + "config": asdict(self), + "tables": tables, + "history_start_us": int(start.timestamp()) * 1000000, + "entity_table": "leads" if self.preset == "marketo_v1" else "users", + "label_table": "engagements" + if self.preset == "marketo_v1" + else "interactions", + } + + +def relation(column, table, key, *, active=1.0, null=0.0, distribution="hotspot"): + return { + "column": column, + "table": table, + "key": key, + "active_fraction": active, + "null_fraction": null, + "distribution": distribution, + "hot_fraction": 0.05, + "hot_share": 0.8, + "power": 3.0, + } + + +def marketo_tables(): + spec = json.loads(files("kumo.fm._dataset_support").joinpath("presets/marketo.json").read_text()) + result = {} + for table in spec["tables"]: + name = ALIASES[table["name"]] + g = table.get("graph") + columns = table["columns"] + if g: + columns = [ + {"name": "_row_id", "type": "int64", "role": "primary_key"}, + *columns, + ] + result[name] = { + "base_rows": g["num_src_nodes"] if g else table["rows"], + "primary_key": "_row_id" if g else "Id", + "time_column": table["time_column"], + "columns": columns, + "fkeys": [ + relation( + next(c["name"] for c in columns if c["role"] == "lead_foreign_key"), + "leads", + "Id", + active=g["num_dst_nonzero_degree"] / spec["lead_rows"], + null=1 - g["edge_count"] / g["num_src_nodes"], + distribution="power", + ) + ] + if g + else [], + } + # Stable schema ordering, independent of the source SPEC's table order. + return {name: result[name] for name in ALIASES.values()} + + +def commerce_tables(): + def col(name, dtype, role): + return {"name": name, "type": dtype, "role": role} + + def table(n, key, columns, fkeys=(), time=None): + return { + "base_rows": n, + "primary_key": key, + "time_column": time, + "columns": [col(key, "int64", "primary_key"), *columns], + "fkeys": list(fkeys), + } + + return { + "users": table(6000000, "user_id", [col("region", "string", "categorical")]), + "interactions": table( + 600000000, + "interaction_id", + [ + col("user_id", "int64", "foreign_key"), + col("item_id", "int64", "foreign_key"), + col("event_time", "time", "timestamp"), + col("amount", "float64", "numeric"), + col("event_type", "string", "categorical"), + ], + [ + relation("user_id", "users", "user_id"), + relation("item_id", "items", "item_id"), + ], + "event_time", + ), + "items": table( + 1000000, + "item_id", + [ + col("category_id", "int64", "foreign_key"), + col("price", "float64", "numeric"), + ], + [ + relation( + "category_id", "categories", "category_id", distribution="uniform" + ) + ], + ), + "categories": table( + 10000, "category_id", [col("name", "string", "categorical")] + ), + } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py new file mode 100644 index 000000000..ad4dea4d8 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py @@ -0,0 +1,339 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Stateless vectorized generation and bounded Parquet writing.""" + +import hashlib +import json +import math +import shutil +import time +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .storage import ColumnDigests, canonical_hash, file_digest, write_json + +DAY_SECONDS = 86400 +DAY_US = DAY_SECONDS * 1000000 +TYPES = { + "int64": pa.int64(), + "float64": pa.float64(), + "bool": pa.bool_(), + "string": pa.string(), + "time": pa.timestamp("us", tz="UTC"), + "date": pa.date32(), +} + + +def salt(text): + return int.from_bytes(hashlib.sha256(text.encode()).digest()[:8], "little") + + +def random_bits(ids, key): + """SplitMix64 with wraparound, keyed by a stable column/relation namespace.""" + with np.errstate(over="ignore"): + x = np.asarray(ids, dtype=np.uint64) + np.uint64(key) + x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) + x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) + return x ^ (x >> np.uint64(31)) + + +def uniform(ids, key): + return (random_bits(ids, key) >> np.uint64(11)).astype(np.float64) / 2**53 + + +def prefix(values, text): + return pc.binary_join_element_wise( + pa.scalar(text), pc.cast(pa.array(values), pa.string()), "" + ) + + +def lead_keys(ids): + return pc.binary_join_element_wise( + "L", pc.utf8_lpad(pc.cast(pa.array(ids), pa.string()), 12, "0"), "" + ) + + +def schema_for(table, config): + fkeys = {r["column"]: r for r in table["fkeys"]} + fields = [] + for col in table["columns"]: + nullable = ( + fkeys[col["name"]]["null_fraction"] > 0 + if col["name"] in fkeys + else col["role"] not in ("primary_key", "timestamp") + and config.feature_null_fraction > 0 + ) + fields.append(pa.field(col["name"], TYPES[col["type"]], nullable=nullable)) + return pa.schema(fields) + + +def fk_values(ids, relation, parent_rows, key): + active = relation["active_parents"] + linked = ids < relation["linked_rows"] + if not active: + return np.zeros(len(ids), dtype=np.int64), ~linked + u = uniform(ids, key) + if relation["distribution"] == "power": + tail = np.floor(u ** relation["power"] * active).astype(np.int64) + elif relation["distribution"] == "hotspot": + hot = max(1, min(active, round(active * relation["hot_fraction"]))) + choices = random_bits(ids, key ^ salt("choices")) + if hot == active: + tail = (choices % np.uint64(active)).astype(np.int64) + else: + tail = np.where( + u < relation["hot_share"], + choices % np.uint64(hot), + hot + choices % np.uint64(active - hot), + ).astype(np.int64) + else: + tail = (random_bits(ids, key) % np.uint64(active)).astype(np.int64) + ordinal = np.where(ids < active, ids, tail) + # A bijection over all parents spreads active parents without collisions. + # Keep multiplication within int64, even for a very large parent population. + # Selecting a small coprime multiplier avoids object arrays in the hot path. + multiplier = 65537 + while math.gcd(multiplier, parent_rows) != 1: + multiplier += 2 + if parent_rows > (2**63 - 1) // multiplier: + raise ValueError("Parent population too large for the generator key mapping") + values = (ordinal * multiplier + key % parent_rows) % parent_rows + return values, ~linked + + +def batch(recipe, name, start, size): + config = Config(**recipe["config"]) + table = recipe["tables"][name] + ids = np.arange(start, start + size, dtype=np.int64) + namespace = config.seed ^ salt(name) + fkeys = {} + for rel in table["fkeys"]: + fkeys[rel["column"]] = fk_values( + ids, + rel, + recipe["tables"][rel["table"]]["rows"], + namespace ^ salt(rel["column"]), + ) + event = random_bits(ids, namespace ^ salt("event-time")) % np.uint64( + config.history_days * DAY_SECONDS + ) + recent = uniform(ids, namespace ^ salt("recent")) < config.recent_share + recent_days = min(90, config.history_days) + event = np.where( + recent, + (config.history_days - recent_days) * DAY_SECONDS + + event % np.uint64(recent_days * DAY_SECONDS), + event, + ).astype(np.int64) + # Draw whole-second values; retain timestamp[us] as the physical schema. + event = event * 1000000 + recipe["history_start_us"] + arrays = [] + for col in table["columns"]: + cname, dtype, role = col["name"], col["type"], col["role"] + key = namespace ^ salt(cname) + nulls = None + if role == "primary_key": + value = lead_keys(ids) if dtype == "string" else pa.array(ids) + elif cname in fkeys: + values, nulls = fkeys[cname] + value = lead_keys(values) if dtype == "string" else pa.array(values) + elif dtype == "time": + value = pa.array(event, type=TYPES[dtype]) + elif dtype == "date": + days = event // DAY_US + if cname.startswith("first_"): + days = days - (random_bits(ids, key) % 365).astype(np.int64) - 1 + value = pa.array(days.astype(np.int32), type=pa.date32()) + elif dtype == "bool": + value = pa.array(uniform(ids, key) < (0.2 if cname == "is_mql" else 0.05)) + elif dtype == "int64": + # Preserve cumulative Marketo window shapes without Spark dependencies. + root = ( + cname.rsplit("_", 1)[0] + if cname.endswith(("_30", "_60", "_90")) + else cname + ) + terms = [ + random_bits(ids, namespace ^ salt(root + suffix)) % modulus + for suffix, modulus in [("a", 5), ("b", 4), ("c", 3), ("d", 6)] + ] + count = {"30": 1, "60": 2, "90": 3}.get(cname.rsplit("_", 1)[-1], 4) + values = sum(terms[:count]).astype(np.int64) + if cname.startswith("days_since"): + values = (random_bits(ids, key) % 366).astype(np.int64) + value = pa.array(values) + elif dtype == "float64": + value = pa.array(uniform(ids, key) * 1000) + elif role == "text": + # Bounded-cardinality text; lengths/cardinality are explicit compression controls. + values = random_bits(ids, key) % config.category_cardinality + value = pc.utf8_slice_codeunits( + pc.utf8_rpad(prefix(values, "text_"), config.text_length, "x"), + 0, + config.text_length, + ) + else: + identity = ( + fkeys.get("marketo_lead_id", (ids, None))[0] + if cname.startswith(("account_", "audience_")) + else ids + ) + cardinality = config.category_cardinality + if config.preset == "marketo_v1" and not cname.startswith("extra_"): + cardinality = ( + 250000 + if cname in ("account_id", "account_name", "company") + else ( + 5000 if cname in ("tactic_name", "tactic_program_name") else 12 + ) + ) + value = prefix(random_bits(identity, key) % cardinality, cname + "_") + if role not in ("primary_key", "timestamp") and cname not in fkeys: + nulls = uniform(ids, key ^ salt("nulls")) < config.feature_null_fraction + if nulls is not None: + value = pc.if_else(pa.array(nulls), pa.scalar(None, type=value.type), value) + arrays.append(value) + return pa.RecordBatch.from_arrays(arrays, schema=schema_for(table, config)) + + +class Budget: + def __init__(self, root, config): + self.root, self.config = root, config + + def check(self): + size = sum(p.stat().st_size for p in self.root.rglob("*") if p.is_file()) + free = shutil.disk_usage(self.root).free + if size > self.config.max_output_gib * 2**30: + raise RuntimeError( + "Output disk budget exceeded; incomplete bundle retained for inspection" + ) + if free < self.config.min_free_gib * 2**30: + raise RuntimeError( + "Free-space reserve reached; incomplete bundle retained for inspection" + ) + return size + + +def plan(config): + recipe = config.resolve() + # An uncompressed estimate, not a promised compressed footprint or peak-RSS bound. + estimated = 0 + for table in recipe["tables"].values(): + for col in table["columns"]: + estimated += table["rows"] * ( + config.text_length + 8 + if col["role"] == "text" + else 40 + if col["type"] == "string" + else 9 + ) + return { + "preset": config.preset, + "rows": {k: v["rows"] for k, v in recipe["tables"].items()}, + "edges": sum( + r["linked_rows"] for t in recipe["tables"].values() for r in t["fkeys"] + ), + "estimated_uncompressed_bytes": estimated, + "recipe": recipe, + } + + +def generate(root: Path, config: Config): + from .tasks import build_tasks + from .validate import validate + + root = root.resolve() + recipe = config.resolve() # Fail before creating output for invalid configurations. + root.mkdir(parents=True, exist_ok=False) + budget = Budget(root, config) + budget.check() + started = time.monotonic() + manifest = { + "format": "fm-dataset-v1", + "recipe": recipe, + "tables": {}, + "tasks": {}, + "created_utc": datetime.now(UTC).isoformat(), + "versions": {"numpy": np.__version__, "pyarrow": pa.__version__}, + } + for name, table in recipe["tables"].items(): + directory = root / "tables" / name + directory.mkdir(parents=True) + schema = schema_for(table, config) + digests = ColumnDigests(schema) + inventory = [] + for shard, file_start in enumerate( + range(0, max(1, table["rows"]), config.file_rows) + ): + path = directory / f"part-{shard:05d}.parquet" + stop = min(file_start + config.file_rows, table["rows"]) + with pq.ParquetWriter(path, schema, compression="zstd") as writer: + for group_start in range(file_start, stop, config.row_group_rows): + group_stop = min(stop, group_start + config.row_group_rows) + chunks = [] + for start in range(group_start, group_stop, config.batch_rows): + chunk = batch( + recipe, + name, + start, + min(config.batch_rows, group_stop - start), + ) + digests.update(chunk) + chunks.append(chunk) + writer.write_table( + pa.Table.from_batches(chunks), + row_group_size=group_stop - group_start, + ) + budget.check() + meta = pq.ParquetFile(path).metadata + inventory.append( + { + "path": path.relative_to(root).as_posix(), + "bytes": path.stat().st_size, + "sha256": file_digest(path), + "row_groups": [ + meta.row_group(i).num_rows for i in range(meta.num_row_groups) + ], + } + ) + budget.check() + manifest["tables"][name] = { + "rows": table["rows"], + "schema": schema.to_string(), + "logical_sha256": digests.hexdigest(), + "files": inventory, + } + print( + json.dumps({"table": name, "rows": table["rows"], "files": len(inventory)}), + flush=True, + ) + manifest["logical_id"] = canonical_hash( + { + name: {k: v[k] for k in ("rows", "schema", "logical_sha256")} + for name, v in manifest["tables"].items() + } + ) + manifest["snapshot_id"] = canonical_hash(manifest["tables"]) + write_json(root / "manifest.json", manifest) + build_tasks(root, manifest, budget) + manifest["generation_seconds"] = time.monotonic() - started + write_json(root / "manifest.json", manifest) + report = validate(root, allow_incomplete=True) + write_json(root / "validation.json", report) + budget.check() + write_json( + root / "_COMPLETE.json", + { + "manifest_sha256": file_digest(root / "manifest.json"), + "validation_sha256": file_digest(root / "validation.json"), + }, + ) + return manifest diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json new file mode 100644 index 000000000..07f4553da --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json @@ -0,0 +1,504 @@ +{ + "description": "Synthetic version of the supplied lead-centered graph; source table counts and exact job edge stats.", + "graph_leads": 6889362, + "lead_rows": 6923193, + "default_history_start": "2023-01-01", + "default_history_days": 1095, + "tables": [ + { + "name": "v_qual_notes", + "rows": 81751, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "source_creation_timestamp", + "type": "time", + "role": "timestamp" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + } + ], + "time_column": "source_creation_timestamp", + "graph": { + "edge_count": 77474, + "num_src_nodes": 81446, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 61107 + } + }, + { + "name": "engagements_v2", + "rows": 437528829, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "engagement_datetime", + "type": "time", + "role": "timestamp" + }, + { + "name": "engagement_type", + "type": "string", + "role": "categorical" + }, + { + "name": "engagement_event_asset", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_title", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_persona", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_region", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_sub_region", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_country", + "type": "string", + "role": "categorical" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + }, + { + "name": "account_name", + "type": "string", + "role": "categorical" + }, + { + "name": "account_dbx", + "type": "string", + "role": "categorical" + }, + { + "name": "account_company_size", + "type": "string", + "role": "categorical" + }, + { + "name": "account_industry", + "type": "string", + "role": "categorical" + }, + { + "name": "account_vertical", + "type": "string", + "role": "categorical" + }, + { + "name": "account_sales_region", + "type": "string", + "role": "categorical" + }, + { + "name": "account_business_unit", + "type": "string", + "role": "categorical" + }, + { + "name": "account_spend_tier", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_name", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_program_name", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_product_focus", + "type": "string", + "role": "categorical" + } + ], + "time_column": "engagement_datetime", + "graph": { + "edge_count": 378563179, + "num_src_nodes": 435519698, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 4724442 + } + }, + { + "name": "v_marketo_lead", + "rows": 6923193, + "columns": [ + { + "name": "Id", + "type": "string", + "role": "primary_key" + } + ], + "time_column": null + }, + { + "name": "marketing_kumo_bronze_v_marketo_u2_conversions", + "rows": 609072, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "u2_stage_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + }, + { + "name": "audience_role", + "type": "string", + "role": "categorical" + }, + { + "name": "seniority_level", + "type": "string", + "role": "categorical" + }, + { + "name": "company", + "type": "string", + "role": "categorical" + }, + { + "name": "account_name", + "type": "string", + "role": "categorical" + } + ], + "time_column": "u2_stage_date", + "graph": { + "edge_count": 384578, + "num_src_nodes": 605830, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 139119 + } + }, + { + "name": "marketing_kumo_bronze_v_test_marketo_trigger_activities", + "rows": 20918304, + "columns": [ + { + "name": "id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "activity_date", + "type": "time", + "role": "timestamp" + } + ], + "time_column": "activity_date", + "graph": { + "edge_count": 14773740, + "num_src_nodes": 20841579, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 4041373 + } + }, + { + "name": "marketing_kumo_bronze_rpt_les_metrics_marketo_agg", + "rows": 180883041, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "engagement_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_engagement", + "type": "int64", + "role": "categorical" + }, + { + "name": "is_nnn", + "type": "bool", + "role": "categorical" + }, + { + "name": "is_mql", + "type": "bool", + "role": "categorical" + }, + { + "name": "is_qual_notes", + "type": "bool", + "role": "categorical" + }, + { + "name": "first_nnn_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "first_mql_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "first_qual_notes_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "days_since_last_nnn", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_mql", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_qual_notes", + "type": "int64", + "role": "categorical" + }, + { + "name": "registration_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_registration", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_attendance", + "type": "int64", + "role": "categorical" + }, + { + "name": "trigger_date", + "type": "date", + "role": "timestamp" + } + ], + "time_column": "trigger_date", + "graph": { + "edge_count": 133735759, + "num_src_nodes": 180818601, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 5078899 + } + }, + { + "name": "marketing_kumo_bronze_rpt_les_metrics_marketo_usecase_agg", + "rows": 9281224, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "associated_u1_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u1_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u2_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u2_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u3_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u3_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u6_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u6_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "trigger_date", + "type": "date", + "role": "timestamp" + } + ], + "time_column": "trigger_date", + "graph": { + "edge_count": 5855118, + "num_src_nodes": 9274298, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 115612 + } + } + ] +} diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py new file mode 100644 index 000000000..3fb76bc42 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py @@ -0,0 +1,608 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned, streaming adapters for the original RelBench F1 and H&M databases.""" + +import json +import re +import shutil +import urllib.request +from dataclasses import asdict +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .generate import TYPES, Budget +from .storage import ( + ColumnDigests, + canonical_hash, + file_digest, + quote, + safe_path, + write_json, +) + +REPOSITORY = "stanford-star/relbench-v1" +REVISION = "21d7284df2a9621c71f21bc4a1a30b7e8effb8dd" +TASKS = { + "rel-f1": {"driver-dnf": 30, "driver-position": 60}, + "rel-hm": {"user-churn": 7}, +} +ROW_ID = "_fm_source_row_id" + + +def yaml_read(path): + try: + import yaml + except ImportError as exc: + raise RuntimeError( + "RelBench imports require PyYAML: install fm-dataloader[relbench]" + ) from exc + + return yaml.safe_load(path.read_text()) + + +def fetch(cache, relative, budget): + """Reuse only checksum-verified files under the immutable revision namespace.""" + path = safe_path(cache, relative) + receipt = path.with_suffix(path.suffix + ".sha256") + if path.exists() and receipt.exists(): + if file_digest(path) != receipt.read_text().strip(): + raise ValueError(f"Corrupt cached source: {path}") + return path + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".partial") + url = f"https://huggingface.co/datasets/{REPOSITORY}/resolve/{REVISION}/{relative}" + with ( + urllib.request.urlopen(url, timeout=120) as response, + temporary.open("wb") as stream, + ): + while chunk := response.read(8 * 2**20): + stream.write(chunk) + budget.check() + temporary.replace(path) + receipt.write_text(file_digest(path) + "\n") + return path + + +def normalized_type(dtype): + if pa.types.is_dictionary(dtype): + return normalized_type(dtype.value_type) + if pa.types.is_timestamp(dtype): + return pa.timestamp("us", tz="UTC") + if pa.types.is_integer(dtype): + return pa.int64() + if pa.types.is_floating(dtype): + return pa.float64() + if pa.types.is_string(dtype) or pa.types.is_large_string(dtype): + return pa.string() + if pa.types.is_boolean(dtype) or pa.types.is_date32(dtype): + return dtype + raise ValueError(f"Unsupported source type: {dtype}") + + +def imported_schema(table): + return pa.schema( + [pa.field(c["name"], TYPES[c["type"]], c["nullable"]) for c in table["columns"]] + ) + + +def normalize(batch, schema, offset=0): + arrays = [] + for field in schema: + if field.name == ROW_ID: + arrays.append(pa.array(np.arange(offset, offset + len(batch)), pa.int64())) + else: + # safe=True rejects precision loss (including sub-microsecond timestamps). + arrays.append( + pc.cast( + batch.column(batch.schema.get_field_index(field.name)), + field.type, + safe=True, + ) + ) + return pa.RecordBatch.from_arrays(arrays, schema=schema) + + +def artifact(root, path, rows=None): + result = {"path": path.relative_to(root).as_posix(), "sha256": file_digest(path)} + if rows is not None: + result["rows"] = rows + return result + + +def import_relbench(root, cache, dataset, config=None): + """Import the complete database; never independently sample relational tables.""" + from .validate import validate + + if dataset not in TASKS: + raise ValueError(f"Unsupported dataset: {dataset}") + cfg = config or Config() + # Validate resource controls using the existing configuration validator. + cfg.resolve() + if cfg.context_rows < 1 or cfg.prediction_rows < 1: + raise ValueError("Task sample sizes must be positive") + root, cache = Path(root).resolve(), Path(cache).resolve() / REVISION + if root.exists(): + raise FileExistsError(f"Refusing to overwrite {root}") + cache.mkdir(parents=True, exist_ok=True) + source_budget = Budget(cache, cfg) + source_budget.check() + + def get(relative): + return fetch(cache, relative, source_budget) + source_manifest = get(f"{dataset}/manifest.yaml") + meta = yaml_read(source_manifest) + if meta["name"] != dataset or meta["manifest_version"] != 1: + raise ValueError("Unexpected upstream dataset manifest") + root.mkdir(parents=True) + budget = Budget(root, cfg) + source_dir = root / "source" + source_dir.mkdir() + provenance = [] + for relative in ["README.md", f"{dataset}/README.md", f"{dataset}/manifest.yaml"]: + original = get(relative) + destination = source_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(original, destination) + provenance.append(artifact(root, destination)) + recipe = { + "source_kind": "relbench", + "adapter_version": 1, + "config": asdict(cfg), + "dataset": dataset, + "tables": {}, + "repository": REPOSITORY, + "revision": REVISION, + "val_timestamp": meta["val_timestamp"], + "test_timestamp": meta["test_timestamp"], + } + manifest = { + "format": "fm-dataset-v1", + "recipe": recipe, + "tables": {}, + "tasks": {}, + "workloads": [], + "source_artifacts": provenance, + "versions": {"pyarrow": pa.__version__}, + "created_utc": datetime.now(UTC).isoformat(), + } + type_names = {v: k for k, v in TYPES.items()} + for name, spec in meta["tables"].items(): + source = get(f"{dataset}/db/{name}.parquet") + pf = pq.ParquetFile(source) + key = spec["pkey"] or ROW_ID + if ROW_ID in pf.schema_arrow.names: + raise ValueError("Reserved occurrence-ID column already exists") + fields = [ + pa.field(f.name, normalized_type(f.type), f.name != key) + for f in pf.schema_arrow + ] + if spec["pkey"] is None: + fields.insert(0, pa.field(ROW_ID, pa.int64(), False)) + schema = pa.schema(fields) + table = { + "rows": pf.metadata.num_rows, + "primary_key": key, + "time_column": spec["time_col"], + "key_policy": "source-key" + if spec["pkey"] + else "zero-based-pinned-source-row-offset", + "source": { + "path": f"{dataset}/db/{name}.parquet", + "sha256": file_digest(source), + "schema": pf.schema_arrow.remove_metadata().to_string(), + }, + "columns": [ + { + "name": f.name, + "type": type_names[f.type], + "nullable": f.nullable, + "role": "primary_key" + if f.name == key + else "timestamp" + if f.name == spec["time_col"] + else "feature", + } + for f in schema + ], + "fkeys": [ + {"column": col, "table": parent, "key": meta["tables"][parent]["pkey"]} + for col, parent in spec["fkeys"].items() + ], + } + recipe["tables"][name] = table + folder = root / "tables" / name + folder.mkdir(parents=True) + digest, inventory, offset, writer, file_count = ( + ColumnDigests(schema), + [], + 0, + None, + 0, + ) + + def close_part(writer, path, inventory): + writer.close() + m = pq.ParquetFile(path).metadata + inventory.append( + { + **artifact(root, path), + "bytes": path.stat().st_size, + "row_groups": [ + m.row_group(i).num_rows for i in range(m.num_row_groups) + ], + } + ) + + try: + for batch in pf.iter_batches(batch_size=cfg.batch_rows): + chunk = normalize(batch, schema, offset) + digest.update(chunk) + offset += len(chunk) + start = 0 + while start < len(chunk): + if writer is None: + path = folder / f"part-{len(inventory):05d}.parquet" + writer = pq.ParquetWriter(path, schema, compression="zstd") + file_count = 0 + count = min(len(chunk) - start, cfg.file_rows - file_count) + writer.write_batch( + chunk.slice(start, count), row_group_size=cfg.row_group_rows + ) + start += count + file_count += count + if file_count == cfg.file_rows: + close_part(writer, path, inventory) + writer = None + budget.check() + if writer is None and not inventory: + path = folder / "part-00000.parquet" + writer = pq.ParquetWriter(path, schema, compression="zstd") + finally: + if writer is not None: + close_part(writer, path, inventory) + manifest["tables"][name] = { + "rows": offset, + "schema": schema.to_string(), + "logical_sha256": digest.hexdigest(), + "files": inventory, + } + print( + json.dumps({"dataset": dataset, "table": name, "rows": offset}), flush=True + ) + manifest["logical_id"] = canonical_hash( + { + n: {k: t[k] for k in ("rows", "schema", "logical_sha256")} + for n, t in manifest["tables"].items() + } + ) + manifest["snapshot_id"] = canonical_hash(manifest["tables"]) + build_imported_tasks(root, manifest, get, budget) + write_json(root / "manifest.json", manifest) + report = validate(root, allow_incomplete=True) + write_json(root / "validation.json", report) + budget.check() + write_json( + root / "_COMPLETE.json", + { + "manifest_sha256": file_digest(root / "manifest.json"), + "validation_sha256": file_digest(root / "validation.json"), + }, + ) + return manifest + + +def task_rows(path, spec, split, limit=None, cutoff=None): + """Read bounded samples in pinned source order, retaining source row identity.""" + rows, offset = [], 0 + horizon = timedelta(days=spec["horizon_days"]) + for batch in pq.ParquetFile(path).iter_batches(batch_size=65536): + for row in batch.to_pylist(): + row_id = offset + offset += 1 + anchor = row[spec["time_col"]] + anchor = ( + anchor.replace(tzinfo=UTC) + if anchor.tzinfo is None + else anchor.astimezone(UTC) + ) + if cutoff is not None and anchor + horizon > cutoff: + continue + target = row[spec["target_col"]] + rows.append( + { + "instance_id": row_id + + {"train": 0, "val": 10**12, "test": 2 * 10**12}[split], + "entity_key": row[spec["entity_col"]], + "anchor_time": anchor, + "target": target, + "label_available_time": anchor + horizon, + } + ) + if limit is not None and len(rows) == limit: + return rows + return rows + + +def build_imported_tasks(root, manifest, get, budget): + recipe, cfg = manifest["recipe"], manifest["recipe"]["config"] + dataset = recipe["dataset"] + for task, days in TASKS[dataset].items(): + relative = f"{dataset}/tasks/{task}/manifest.yaml" + original = get(relative) + spec = yaml_read(original) + if spec["kind"] != "forecast" or spec["timedelta"] != f"{days} days": + raise ValueError("Unexpected upstream task horizon/kind") + spec["horizon_days"] = days + folder = root / "tasks" / task + folder.mkdir(parents=True) + shutil.copyfile(original, folder / "source-manifest.yaml") + source_spec = artifact(root, folder / "source-manifest.yaml") + splits = {} + for split in ("train", "val", "test"): + source = get(f"{dataset}/tasks/{task}/{split}.parquet") + destination = folder / f"source-{split}.parquet" + shutil.copyfile(source, destination) + splits[split] = artifact( + root, destination, pq.ParquetFile(destination).metadata.num_rows + ) + budget.check() + predictions = task_rows( + root / splits["val"]["path"], spec, "val", cfg["prediction_rows"] + ) + cutoff = min(r["anchor_time"] for r in predictions) + contexts = task_rows( + root / splits["train"]["path"], spec, "train", cfg["context_rows"], cutoff + ) + if not contexts or not predictions: + raise ValueError("No eligible context/prediction instances") + target_type = pa.float64() if spec["task_type"] == "regression" else pa.int64() + key = recipe["tables"][spec["entity_table"]]["primary_key"] + key_type = ( + imported_schema(recipe["tables"][spec["entity_table"]]).field(key).type + ) + schema = pa.schema( + [ + pa.field("instance_id", pa.int64(), False), + pa.field("entity_key", key_type, False), + pa.field("anchor_time", pa.timestamp("us", tz="UTC"), False), + pa.field("target", target_type, False), + pa.field("label_available_time", pa.timestamp("us", tz="UTC"), False), + ] + ) + context, prediction = ( + pa.Table.from_pylist(contexts, schema), + pa.Table.from_pylist(predictions, schema), + ) + artifacts = {} + for kind, data in [ + ("context", context), + ( + "predict", + prediction.select(["instance_id", "entity_key", "anchor_time"]), + ), + ("labels", prediction.select(["instance_id", "target"])), + ]: + path = folder / f"{kind}.parquet" + pq.write_table(data, path, compression="zstd") + artifacts[kind] = artifact(root, path, len(data)) + manifest["tasks"][task] = { + "type": "regression" if target_type == pa.float64() else "classification", + "entity_table": spec["entity_table"], + "label_interval": f"(anchor, anchor + {days} days]", + "source_spec": source_spec, + "source_splits": splits, + "source_metadata": spec, + "prediction_split": "val", + "selection": "first-eligible-pinned-source-rows-v1", + "artifacts": artifacts, + } + for size in sorted( + {min(n, len(prediction)) for n in (1, 32, cfg["prediction_rows"])} + ): + paths = ( + [ + ["drivers", "results", "races", "circuits"], + ["drivers", "results", "constructors"], + ] + if dataset == "rel-f1" + else [["customer", "transactions", "article"]] + ) + work = { + "task": task, + "logical_id": manifest["logical_id"], + "snapshot_id": manifest["snapshot_id"], + "cohort": "official-val", + "context_instance_ids": context["instance_id"].to_pylist(), + "prediction_instance_ids": prediction["instance_id"].to_pylist()[:size], + "paths": paths, + "event_fanout": 16, + "forward_fk_fanout": 1, + "lookback_days": cfg["lookback_days"], + "history_interval": "[anchor-lookback, anchor)", + "event_order": ["timestamp DESC", "primary_key ASC"], + "timeless_dimensions": [ + n for n, t in recipe["tables"].items() if not t["time_column"] + ], + "identity_policy": "deduplicate source IDs within each instance; preserve edges and per-instance occurrences", + "projection": { + n: [c["name"] for c in t["columns"]] + for n, t in recipe["tables"].items() + }, + "feature_columns": { + n: [ + c["name"] + for c in t["columns"] + if c["role"] != "primary_key" + and c["name"] not in {r["column"] for r in t["fkeys"]} + ] + for n, t in recipe["tables"].items() + }, + } + work["workload_id"] = canonical_hash(work) + path = root / "workloads" / f"{task}-official-val-{size}.json" + path.parent.mkdir(exist_ok=True) + write_json(path, work) + manifest["workloads"].append(artifact(root, path)) + + +def validate_recipe(root, manifest): + recipe = manifest["recipe"] + if ( + recipe["repository"] != REPOSITORY + or recipe["revision"] != REVISION + or recipe["adapter_version"] != 1 + ): + raise ValueError("Unrecognized RelBench source pin/adapter") + dataset = recipe["dataset"] + if dataset not in TASKS or set(manifest["tasks"]) != set(TASKS[dataset]): + raise ValueError("Unexpected RelBench dataset/tasks") + for item in manifest["source_artifacts"]: + if file_digest(safe_path(root, item["path"])) != item["sha256"]: + raise ValueError("Source metadata checksum mismatch") + source = yaml_read(root / "source" / dataset / "manifest.yaml") + if set(source["tables"]) != set(recipe["tables"]): + raise ValueError("Source table inventory mismatch") + for name, spec in source["tables"].items(): + table = recipe["tables"][name] + expected = [ + {"column": col, "table": parent, "key": source["tables"][parent]["pkey"]} + for col, parent in spec["fkeys"].items() + ] + if ( + table["primary_key"] != (spec["pkey"] or ROW_ID) + or table["time_column"] != spec["time_col"] + or table["fkeys"] != expected + ): + raise ValueError("Source relational metadata mismatch") + if not re.fullmatch(r"[0-9a-f]{64}", table["source"]["sha256"]): + raise ValueError("Missing source checksum") + for key in ("val_timestamp", "test_timestamp"): + if recipe[key] != source[key]: + raise ValueError("Source split cutoff mismatch") + return recipe + + +def validate_tasks(root, manifest, con, report): + instance_sets = {} + recipe, cfg = manifest["recipe"], manifest["recipe"]["config"] + for task, task_spec in manifest["tasks"].items(): + source_spec = task_spec["source_spec"] + path = safe_path(root, source_spec["path"]) + if file_digest(path) != source_spec["sha256"]: + raise ValueError("Source task metadata checksum mismatch") + spec = yaml_read(path) + spec["horizon_days"] = TASKS[recipe["dataset"]][task] + if ( + spec != task_spec["source_metadata"] + or spec["timedelta"] != f"{spec['horizon_days']} days" + ): + raise ValueError("Source task specification mismatch") + if ( + task_spec["entity_table"] != spec["entity_table"] + or task_spec["prediction_split"] != "val" + ): + raise ValueError("Task mapping mismatch") + if set(task_spec["source_splits"]) != {"train", "val", "test"}: + raise ValueError("Missing official task splits") + for split, item in task_spec["source_splits"].items(): + path = safe_path(root, item["path"]) + pf = pq.ParquetFile(path) + if ( + file_digest(path) != item["sha256"] + or pf.metadata.num_rows != item["rows"] + ): + raise ValueError("Official task split checksum/count mismatch") + con.read_parquet(str(path)).create_view("official_split", replace=True) + entity, col, stamp, target = map( + quote, + [ + spec["entity_table"], + spec["entity_col"], + spec["time_col"], + spec["target_col"], + ], + ) + pk = quote(recipe["tables"][spec["entity_table"]]["primary_key"]) + missing = con.execute( + f"SELECT count(*) FROM official_split s ANTI JOIN {entity} e ON s.{col}=e.{pk}" + ).fetchall()[0][0] + nulls = con.execute( + f"SELECT count(*) FROM official_split WHERE {col} IS NULL OR {stamp} IS NULL OR {target} IS NULL" + ).fetchall()[0][0] + val, test = recipe["val_timestamp"], recipe["test_timestamp"] + condition = ( + f"{stamp} >= ?::TIMESTAMP" + if split == "train" + else f"{stamp} < ?::TIMESTAMP OR {stamp} >= ?::TIMESTAMP" + if split == "val" + else f"{stamp} < ?::TIMESTAMP" + ) + params = ( + [val] if split == "train" else [val, test] if split == "val" else [test] + ) + wrong_split = con.execute( + f"SELECT count(*) FROM official_split WHERE {condition}", params + ).fetchall()[0][0] + if missing or nulls or wrong_split: + raise ValueError( + f"Invalid official split entities/labels/cutoffs: {task}/{split}" + ) + predictions = task_rows( + safe_path(root, task_spec["source_splits"]["val"]["path"]), + spec, + "val", + cfg["prediction_rows"], + ) + cutoff = min(r["anchor_time"] for r in predictions) + contexts = task_rows( + safe_path(root, task_spec["source_splits"]["train"]["path"]), + spec, + "train", + cfg["context_rows"], + cutoff, + ) + expected = { + "context": contexts, + "predict": [ + {k: r[k] for k in ("instance_id", "entity_key", "anchor_time")} + for r in predictions + ], + "labels": [ + {k: r[k] for k in ("instance_id", "target")} for r in predictions + ], + } + if set(task_spec["artifacts"]) != set(expected): + raise ValueError("Missing task artifacts") + for kind, item in task_spec["artifacts"].items(): + path = safe_path(root, item["path"]) + data = pq.read_table(path) + if ( + file_digest(path) != item["sha256"] + or len(data) != item["rows"] + or data.to_pylist() != expected[kind] + ): + raise ValueError( + f"Task differs from official labels/selection: {task}/{kind}" + ) + instance_sets[task] = { + kind: {r["instance_id"] for r in expected[kind]} + for kind in ("context", "predict") + } + report["tasks"][task] = { + "source_splits": { + k: v["rows"] for k, v in task_spec["source_splits"].items() + }, + "context": len(contexts), + "predict": len(predictions), + "horizon_days": spec["horizon_days"], + "labels_verified_against": "pinned official splits (not recomputed)", + } + return instance_sets diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py new file mode 100644 index 000000000..d20d5afc8 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py @@ -0,0 +1,263 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Batch-independent logical hashes and portable artifact inventories.""" + +import hashlib +import json +import shutil +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + + +def canonical_hash(value): + return hashlib.sha256( + json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + +def file_digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def write_json(path, value): + Path(path).write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n" + ) + + +def safe_path(root, relative): + path = (root / relative).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError(f"Artifact path escapes bundle: {relative}") + return path + + +class ColumnDigests: + """Version 1: independent validity/value/length streams in PK source order. + + Separating streams makes the result independent of writer/reader chunk sizes. + Null values use zero/empty bytes; validity distinguishes null from those values. + """ + + def __init__(self, schema): + self.schema = schema + self.streams = [[hashlib.sha256() for _ in range(3)] for _ in schema] + + def update(self, batch): + for field, array, (valid, values, lengths) in zip( + self.schema, batch.columns, self.streams, strict=True + ): + valid.update(np.asarray(pc.is_valid(array)).astype(np.uint8).tobytes()) + if pa.types.is_string(field.type): + filled = pc.fill_null(array, "") + offsets = np.frombuffer(filled.buffers()[1], dtype=" int64 epoch microseconds; no temporal adjacency index", + "derived_files": inventory, + "seconds": time.monotonic() - start, + "ingest_config_sha256": file_digest(output / "ingest.json"), + }, + ) + budget.check() + write_json( + output / "_COMPLETE.json", + {"preparation_sha256": file_digest(output / "preparation.json")}, + ) + return config diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py new file mode 100644 index 000000000..319b1c40f --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Freeze cohorts, context, and future labels outside payload timing.""" + +from datetime import UTC, datetime + +import pyarrow as pa +import pyarrow.parquet as pq + +from .generate import DAY_US +from .storage import canonical_hash, connect, file_digest, quote, write_json + + +def as_time(microseconds): + return datetime.fromtimestamp(microseconds / 1000000, UTC) + + +def label_counts(con, manifest, instances): + """Count target events in [anchor, anchor+30d), including zero-event entities.""" + recipe = manifest["recipe"] + table = recipe["tables"][recipe["label_table"]] + entity = recipe["entity_table"] + fk = next(r["column"] for r in table["fkeys"] if r["table"] == entity) + con.register( + "label_instances", + instances.select(["instance_id", "entity_key", "anchor_time"]), + ) + try: + return con.sql(f""" + SELECT a.instance_id, count(f.{quote(table["primary_key"])})::BIGINT AS target + FROM label_instances a LEFT JOIN {quote(recipe["label_table"])} f + ON a.entity_key = f.{quote(fk)} + AND f.{quote(table["time_column"])} >= a.anchor_time + AND f.{quote(table["time_column"])} < a.anchor_time + INTERVAL 30 DAY + GROUP BY a.instance_id ORDER BY a.instance_id + """).to_arrow_table() + finally: + con.unregister("label_instances") + + +def build_tasks(root, manifest, budget): + cfg = manifest["recipe"]["config"] + recipe = manifest["recipe"] + pred_us = recipe["history_start_us"] + (cfg["history_days"] - 30) * DAY_US + ctx_us = pred_us - 30 * DAY_US + entity = recipe["entity_table"] + entity_key = recipe["tables"][entity]["primary_key"] + con = connect(root, manifest) + try: + con.execute("SET TimeZone='UTC'").fetchall() + parts = [] + for name, table in recipe["tables"].items(): + for rel in table["fkeys"]: + if rel["table"] == entity and table["time_column"]: + parts.append(f"""SELECT {quote(rel["column"])} AS entity_key FROM {quote(name)} + WHERE {quote(table["time_column"])} >= make_timestamp({pred_us - cfg["lookback_days"] * DAY_US}) + AND {quote(table["time_column"])} < make_timestamp({pred_us}) + AND {quote(rel["column"])} IS NOT NULL""") + con.execute(f"""CREATE TEMP TABLE degrees AS + SELECT e.{quote(entity_key)} AS entity_key, coalesce(d.degree,0)::BIGINT AS degree + FROM {quote(entity)} e LEFT JOIN + (SELECT entity_key, count(*) degree FROM ({" UNION ALL ".join(parts)}) GROUP BY entity_key) d + ON e.{quote(entity_key)} = d.entity_key""").fetchall() + median = con.execute( + "SELECT approx_quantile(degree, 0.5) FROM degrees" + ).fetchall()[0][0] + contexts = con.execute(f"""SELECT entity_key FROM degrees + ORDER BY hash(entity_key, {cfg["seed"]}), entity_key LIMIT {cfg["context_rows"]}""").fetchall() + pred_records, cohorts = [], {} + for cohort, order in [ + ("quiet", "degree ASC"), + ("typical", f"abs(degree - {median}) ASC"), + ("high_degree", "degree DESC"), + ]: + rows = con.execute(f"""SELECT entity_key, degree FROM degrees + ORDER BY {order}, hash(entity_key, {cfg["seed"]}), entity_key + LIMIT {cfg["prediction_rows"]}""").fetchall() + members = [] + for key, degree in rows: + instance_id = 1000000 + len(pred_records) + pred_records.append( + { + "instance_id": instance_id, + "entity_key": key, + "anchor_time": as_time(pred_us), + } + ) + members.append( + {"instance_id": instance_id, "entity_key": key, "degree": degree} + ) + cohorts[cohort] = members + key_type = pa.string() if cfg["preset"] == "marketo_v1" else pa.int64() + base_schema = pa.schema( + [ + pa.field("instance_id", pa.int64(), False), + pa.field("entity_key", key_type, False), + pa.field("anchor_time", pa.timestamp("us", tz="UTC"), False), + ] + ) + context = pa.Table.from_pylist( + [ + {"instance_id": i, "entity_key": row[0], "anchor_time": as_time(ctx_us)} + for i, row in enumerate(contexts) + ], + schema=base_schema, + ) + predict = pa.Table.from_pylist(pred_records, schema=base_schema) + ctx_counts = label_counts(con, manifest, context)["target"].to_pylist() + pred_counts = label_counts(con, manifest, predict)["target"].to_pylist() + degree_stats = con.execute("""SELECT min(degree), approx_quantile(degree, [0.5,0.9,0.99]), + max(degree), count(*) FILTER (WHERE degree > 0) FROM degrees""").fetchall()[0] + manifest["cohorts"] = cohorts + manifest["degree_stats"] = dict( + zip(["min", "quantiles_50_90_99", "max", "active_entities"], degree_stats) + ) + manifest["selection"] = { + "algorithm": "degree-order-with-duckdb-hash-tiebreak-v1", + "seed": cfg["seed"], + "prediction_cutoff": as_time(pred_us).isoformat(), + "context_cutoff": as_time(ctx_us).isoformat(), + } + if cfg["preset"] == "commerce_v1": + con.register("selected_users", predict.select(["entity_key"])) + overlap = con.execute(f"""SELECT count(*), coalesce(max(users),0), + count(*) FILTER (WHERE users > 1) FROM ( + SELECT item_id, count(DISTINCT user_id) users FROM interactions + WHERE user_id IN (SELECT entity_key FROM selected_users) + AND item_id IS NOT NULL AND event_time < make_timestamp({pred_us}) + AND event_time >= make_timestamp({pred_us - cfg["lookback_days"] * DAY_US}) GROUP BY item_id)""").fetchall()[0] + manifest["selected_item_overlap"] = dict( + zip(["items", "max_users_per_item", "items_shared_by_users"], overlap) + ) + for task in ("event_binary", "event_count"): + folder = root / "tasks" / task + folder.mkdir(parents=True) + targets = ( + (lambda values: [int(v > 0) for v in values]) + if task == "event_binary" + else list + ) + ctx = context.append_column( + pa.field("target", pa.int64(), False), + pa.array(targets(ctx_counts), pa.int64()), + ) + ctx = ctx.append_column( + pa.field("label_available_time", pa.timestamp("us", tz="UTC"), False), + pa.array([as_time(pred_us)] * len(ctx), pa.timestamp("us", tz="UTC")), + ) + labels = pa.table( + { + "instance_id": predict["instance_id"], + "target": pa.array(targets(pred_counts), pa.int64()), + } + ) + artifacts = {} + for name, data in [ + ("context", ctx), + ("predict", predict), + ("labels", labels), + ]: + path = folder / f"{name}.parquet" + pq.write_table(data, path, compression="zstd") + artifacts[name] = { + "path": path.relative_to(root).as_posix(), + "sha256": file_digest(path), + "rows": len(data), + } + manifest["tasks"][task] = { + "type": "classification" if task == "event_binary" else "regression", + "entity_table": entity, + "label_table": recipe["label_table"], + "label_interval": "[anchor, anchor + 30 days)", + "artifacts": artifacts, + } + for cohort, members in cohorts.items(): + for size in sorted( + {min(n, len(members)) for n in (1, 32, cfg["prediction_rows"])} + ): + paths = ( + [ + ["leads", "engagements"], + ["leads", "metrics"], + ["leads", "usecase_metrics"], + ["leads", "conversions"], + ["leads", "activities"], + ["leads", "notes"], + ] + if cfg["preset"] == "marketo_v1" + else [ + ["users", "interactions", "items"], + ["users", "interactions", "items", "categories"], + ] + ) + workload = { + "task": task, + "logical_id": manifest["logical_id"], + "snapshot_id": manifest["snapshot_id"], + "cohort": cohort, + "context_instance_ids": context["instance_id"].to_pylist(), + "prediction_instance_ids": [ + r["instance_id"] for r in members[:size] + ], + "paths": paths, + "event_fanout": 16, + "forward_fk_fanout": 1, + "lookback_days": cfg["lookback_days"], + "history_interval": "[anchor-lookback, anchor)", + "event_order": ["timestamp DESC", "primary_key ASC"], + "timeless_dimensions": [ + n + for n, t in recipe["tables"].items() + if not t["time_column"] + ], + "identity_policy": "deduplicate source IDs within each instance; preserve edges and per-instance occurrences", + "projection": { + n: [c["name"] for c in t["columns"]] + for n, t in recipe["tables"].items() + }, + "feature_columns": { + n: [ + c["name"] + for c in t["columns"] + if c["role"] != "primary_key" + and c["name"] not in {r["column"] for r in t["fkeys"]} + ] + for n, t in recipe["tables"].items() + }, + } + workload["workload_id"] = canonical_hash(workload) + path = root / "workloads" / f"{task}-{cohort}-{size}.json" + path.parent.mkdir(exist_ok=True) + write_json(path, workload) + manifest["workloads"] = [ + {"path": p.relative_to(root).as_posix(), "sha256": file_digest(p)} + for p in sorted((root / "workloads").glob("*.json")) + ] + import duckdb + + manifest["versions"]["duckdb"] = duckdb.__version__ + budget.check() + finally: + con.close() + # DuckDB deletes its own temporary files when closed; retain the directory. diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py new file mode 100644 index 000000000..f9c9b974a --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py @@ -0,0 +1,286 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Streaming content checks plus spillable SQL integrity/label validation.""" + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .generate import Budget, schema_for +from .storage import ( + ColumnDigests, + canonical_hash, + connect, + file_digest, + quote, + safe_path, +) +from .tasks import label_counts + + +def validate(root: Path, *, allow_incomplete=False): + root = root.resolve() + manifest = json.loads((root / "manifest.json").read_text()) + if manifest.get("format") != "fm-dataset-v1": + raise ValueError("Unsupported bundle format") + if not allow_incomplete: + marker = json.loads((root / "_COMPLETE.json").read_text()) + for name, key in [ + ("manifest.json", "manifest_sha256"), + ("validation.json", "validation_sha256"), + ]: + if file_digest(root / name) != marker[key]: + raise ValueError(f"Completion checksum mismatch: {name}") + cfg = Config(**manifest["recipe"]["config"]) + imported = manifest["recipe"].get("source_kind") == "relbench" + if imported: + from .relbench import imported_schema, validate_recipe, validate_tasks + + cfg.resolve() + recipe = validate_recipe(root, manifest) + else: + recipe = cfg.resolve() + if recipe != manifest["recipe"] or set(manifest["tables"]) != set(recipe["tables"]): + raise ValueError("Recipe/schema mismatch") + budget = Budget(root, cfg) + report = { + "status": "pass", + "tables": {}, + "tasks": {}, + "logical_id": manifest["logical_id"], + } + for name, info in manifest["tables"].items(): + table = recipe["tables"][name] + schema = imported_schema(table) if imported else schema_for(table, cfg) + if info["schema"] != schema.to_string(): + raise ValueError(f"Manifest schema mismatch: {name}") + digests, count, previous = ColumnDigests(schema), 0, None + nulls = {f.name: 0 for f in schema} + if not info["files"]: + raise ValueError(f"Missing source files: {name}") + for item in info["files"]: + path = safe_path(root, item["path"]) + if ( + file_digest(path) != item["sha256"] + or path.stat().st_size != item["bytes"] + ): + raise ValueError(f"File checksum/size mismatch: {path}") + pf = pq.ParquetFile(path) + groups = [ + pf.metadata.row_group(i).num_rows + for i in range(pf.metadata.num_row_groups) + ] + if groups != item["row_groups"] or any( + n > cfg.row_group_rows for n in groups + ): + raise ValueError(f"Row-group mismatch: {name}") + if not pf.schema_arrow.equals(schema): + raise ValueError(f"Schema mismatch: {name}") + for batch in pf.iter_batches(batch_size=65536): + count += len(batch) + digests.update(batch) + for field, array in zip(schema, batch.columns, strict=True): + nulls[field.name] += array.null_count + if not field.nullable and array.null_count: + raise ValueError(f"Null required value: {name}.{field.name}") + keys = batch.column(schema.get_field_index(table["primary_key"])) + if len(keys) and not imported: + if previous is not None and keys[0].as_py() <= previous: + raise ValueError(f"Primary keys not strictly ordered: {name}") + if ( + len(keys) > 1 + and pc.any( + pc.less_equal(keys.slice(1), keys.slice(0, len(keys) - 1)) + ).as_py() + ): + raise ValueError(f"Duplicate/unordered primary keys: {name}") + previous = keys[-1].as_py() + if ( + count != table["rows"] + or count != info["rows"] + or digests.hexdigest() != info["logical_sha256"] + ): + raise ValueError(f"Row count/logical checksum mismatch: {name}") + report["tables"][name] = { + "rows": count, + "null_counts": nulls, + "relationships": {}, + } + budget.check() + logical = canonical_hash( + { + n: {k: t[k] for k in ("rows", "schema", "logical_sha256")} + for n, t in manifest["tables"].items() + } + ) + if ( + logical != manifest["logical_id"] + or canonical_hash(manifest["tables"]) != manifest["snapshot_id"] + ): + raise ValueError("Dataset identity mismatch") + con = connect(root, manifest) + try: + for name, table in recipe["tables"].items(): + if imported: + pk = quote(table["primary_key"]) + total, unique = con.execute( + f"SELECT count(*), count(DISTINCT {pk}) FROM {quote(name)}" + ).fetchall()[0] + if total != unique: + raise ValueError(f"Duplicate/null primary keys: {name}") + if ( + table["key_policy"] == "zero-based-pinned-source-row-offset" + and total + ): + low, high = con.execute( + f"SELECT min({pk}), max({pk}) FROM {quote(name)}" + ).fetchall()[0] + if (low, high) != (0, total - 1): + raise ValueError("Invalid source occurrence IDs") + for rel in table["fkeys"]: + fk = quote(rel["column"]) + missing = con.execute(f"""SELECT count(*) FROM {quote(name)} f + ANTI JOIN {quote(rel["table"])} p ON f.{fk}=p.{quote(rel["key"])} + WHERE f.{fk} IS NOT NULL""").fetchall()[0][0] + linked, active = con.execute( + f"SELECT count({fk}), count(DISTINCT {fk}) FROM {quote(name)}" + ).fetchall()[0] + if ( + missing + or (not imported and linked != rel["linked_rows"]) + or (not imported and active != rel["active_parents"]) + ): + raise ValueError( + f"Foreign-key integrity/coverage mismatch: {name}.{rel['column']}" + ) + degree = con.execute(f"""SELECT min(n), approx_quantile(n, [0.5,0.9,0.99]), max(n) + FROM (SELECT count(*) n FROM {quote(name)} WHERE {fk} IS NOT NULL GROUP BY {fk})""").fetchall()[0] + report["tables"][name]["relationships"][rel["column"]] = { + "linked_rows": linked, + "active_parents": active, + "missing": missing, + "nonzero_degree_min": degree[0], + "nonzero_degree_quantiles_50_90_99": degree[1], + "nonzero_degree_max": degree[2], + } + if table["time_column"]: + bounds = con.execute( + f"SELECT min({quote(table['time_column'])})::VARCHAR, max({quote(table['time_column'])})::VARCHAR FROM {quote(name)}" + ).fetchall()[0] + report["tables"][name]["time_range"] = list(bounds) + if imported: + instance_sets = validate_tasks(root, manifest, con, report) + else: + instance_sets = {} + if set(manifest["tasks"]) != {"event_binary", "event_count"}: + raise ValueError("Expected both generated tasks") + for task, spec in manifest["tasks"].items(): + data = {} + for kind, item in spec["artifacts"].items(): + path = safe_path(root, item["path"]) + if file_digest(path) != item["sha256"]: + raise ValueError(f"Task checksum mismatch: {task}/{kind}") + data[kind] = pq.read_table(path) + if len(data[kind]) != item["rows"]: + raise ValueError("Task row count mismatch") + ids = data[kind]["instance_id"].to_pylist() + if len(set(ids)) != len(ids): + raise ValueError("Duplicate task instance IDs") + if "target" in data["predict"].column_names: + raise ValueError("Prediction target leakage") + for kind in ("context", "predict"): + instances = data[kind] + if not len(instances) or any( + instances[c].null_count for c in instances.column_names + ): + raise ValueError("Empty task or null instance fields") + con.register("validate_instances", instances) + entity = recipe["entity_table"] + entity_key = recipe["tables"][entity]["primary_key"] + missing = con.execute( + f"SELECT count(*) FROM validate_instances a ANTI JOIN {quote(entity)} e ON a.entity_key=e.{quote(entity_key)}" + ).fetchall()[0][0] + con.unregister("validate_instances") + if missing: + raise ValueError("Task references a missing entity") + history_end = datetime.fromtimestamp( + recipe["history_start_us"] / 1000000, UTC + ) + timedelta(days=cfg.history_days) + if any( + t + timedelta(days=30) > history_end + for t in instances["anchor_time"].to_pylist() + ): + raise ValueError("Incomplete label observation interval") + actual = label_counts(con, manifest, data[kind])[ + "target" + ].to_pylist() + expected = data["context" if kind == "context" else "labels"][ + "target" + ].to_pylist() + if spec["type"] == "classification": + actual = [int(v > 0) for v in actual] + if actual != expected: + raise ValueError(f"Label mismatch: {task}/{kind}") + if ( + data["predict"]["instance_id"].to_pylist() + != data["labels"]["instance_id"].to_pylist() + ): + raise ValueError("Prediction/label alignment mismatch") + if ( + pc.max(data["context"]["label_available_time"]).as_py() + > pc.min(data["predict"]["anchor_time"]).as_py() + ): + raise ValueError("Context labels not available at prediction time") + if any( + available != anchor + timedelta(days=30) + for anchor, available in zip( + data["context"]["anchor_time"].to_pylist(), + data["context"]["label_available_time"].to_pylist(), + strict=True, + ) + ): + raise ValueError("Incorrect context label availability") + instance_sets[task] = { + kind: set(data[kind]["instance_id"].to_pylist()) + for kind in ("context", "predict") + } + report["tasks"][task] = {k: len(v) for k, v in data.items()} + if not manifest["tasks"] or not manifest["workloads"]: + raise ValueError("Missing tasks/workloads") + for item in manifest["workloads"]: + path = safe_path(root, item["path"]) + if file_digest(path) != item["sha256"]: + raise ValueError("Workload checksum mismatch") + work = json.loads(path.read_text()) + identifier = work.pop("workload_id") + if ( + canonical_hash(work) != identifier + or work["snapshot_id"] != manifest["snapshot_id"] + or work["logical_id"] != manifest["logical_id"] + ): + raise ValueError("Workload identity mismatch") + if work["task"] not in instance_sets: + raise ValueError("Unknown workload task") + for split in ("context", "predict"): + key = ( + "context_instance_ids" + if split == "context" + else "prediction_instance_ids" + ) + ids = work[key] + if ( + not ids + or len(set(ids)) != len(ids) + or not set(ids) <= instance_sets[work["task"]][split] + ): + raise ValueError("Workload references missing/duplicate instances") + budget.check() + finally: + con.close() + return report diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py new file mode 100644 index 000000000..0ba98d533 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""HTTP encodings over retained Arrow tables.""" +import base64 + +import pyarrow as pa +import pyarrow.compute as pc + +from . import native_json +from ._phase import phase +from .bundle import TYPES + + +def encode_arrow_table(table, columns, encoding, budget, *, native_arrays=False): + """One final scalar conversion for arrays; Arrow IPC retains typed columns.""" + if len(columns) > budget["max_columns"]: + raise ValueError("Payload table exceeds column budget") + with phase("normalize_wire_values", "type_conversion"): + schema = pa.schema([pa.field(c, TYPES[s["dtype"]], nullable=s["nullable"]) + for c, s in columns.items()]) + table = table.select(list(columns)).cast(schema) + for field in schema: + if not field.nullable and table[field.name].null_count: + raise ValueError(f"Null in required Arrow field: {field.name}") + if encoding == "arrays": + if native_arrays: + # Formatting and serialization are fused; charge the combined work + # to JSON rather than pretending to split a single native call. + with phase("native_arrays_encode", "json"): + rows = native_json.encode_rows(table, budget["max_payload_bytes"]) + return {"format": "arrays", "columns": list(columns), "rows": rows}, 0 + with phase("arrays_conversion", "type_conversion"): + values = [] + for field, column in zip(schema, table.columns, strict=True): + if pa.types.is_timestamp(field.type): + # Arrow's %S retains the timestamp unit's fractional digits. + with phase("arrays_temporal_format", "type_conversion", column=field.name): + column = pc.binary_join_element_wise( + pc.strftime(column, format="%Y-%m-%dT%H:%M:%S"), "Z", "") + elif pa.types.is_date(field.type): + with phase("arrays_temporal_format", "type_conversion", column=field.name): + column = pc.strftime(column, format="%Y-%m-%d") + with phase("arrays_to_python", "type_conversion", column=field.name): + cells = column.to_pylist() + if pa.types.is_int64(field.type): + with phase("arrays_integer_bounds", "type_conversion", column=field.name): + bounds = pc.min_max(column).as_py() + if ((bounds["min"] is not None and bounds["min"] < -(2**53 - 1)) + or (bounds["max"] is not None and bounds["max"] > 2**53 - 1)): + with phase("arrays_large_integer_format", "type_conversion", column=field.name): + cells = [str(v) if v is not None and abs(v) > 2**53 - 1 else v for v in cells] + values.append(cells) + with phase("arrays_row_transpose", "type_conversion"): + return {"format": "arrays", "columns": list(columns), + "rows": list(zip(*values, strict=True))}, 0 + return (table, table.nbytes) if encoding == "arrow" else encode_ipc_table(table) + + +def encode_ipc_table(table): + """Shared IPC/base64 boundary for row and columnar inputs.""" + with phase("arrow_ipc_encode", "arrow_ipc"): + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema, options=pa.ipc.IpcWriteOptions(compression=None)) as writer: + writer.write_table(table) + raw = sink.getvalue().to_pybytes() + with phase("base64_encode", "base64"): + encoded = base64.b64encode(raw).decode() + return {"format": "arrow_ipc", "transport": "base64", "payload_base64": encoded}, len(raw) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp b/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp new file mode 100644 index 000000000..5e5faf932 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp @@ -0,0 +1,238 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +// The GIL stays held: PyOS_double_to_string provides exactly CPython's JSON +// float spelling without constructing a Python float for each Arrow value. +struct PythonError {}; +struct Writer { + std::string data; + size_t limit; + explicit Writer(size_t limit) : limit(limit) {} + void append(const char* s, size_t n) { + if (n > limit - data.size()) + throw std::invalid_argument("Payload exceeds byte budget; reduce the workload batch"); + data.append(s, n); + } + void append(const char* s) { append(s, std::strlen(s)); } + void put(char c) { append(&c, 1); } + void integer(int64_t n) { + char buf[32]; + auto end = std::to_chars(buf, buf + sizeof(buf), n).ptr; + append(buf, static_cast(end - buf)); + } + void hex(uint32_t cp) { + static constexpr char digits[] = "0123456789abcdef"; + char buf[] = {'\\', 'u', digits[(cp >> 12) & 15], digits[(cp >> 8) & 15], + digits[(cp >> 4) & 15], digits[cp & 15]}; + append(buf, sizeof(buf)); + } + // Match json.dumps(ensure_ascii=True), including supplementary pairs. + void string(const char* s, int64_t n) { + put('"'); + int64_t i = 0; + while (i < n) { + int64_t begin = i; + while (i < n && static_cast(s[i]) >= 0x20 && + static_cast(s[i]) < 0x7f && s[i] != '"' && s[i] != '\\') ++i; + append(s + begin, static_cast(i - begin)); + if (i == n) break; + uint32_t cp = static_cast(s[i++]); + switch (cp) { + case '"': append("\\\""); continue; + case '\\': append("\\\\"); continue; + case '\b': append("\\b"); continue; + case '\f': append("\\f"); continue; + case '\n': append("\\n"); continue; + case '\r': append("\\r"); continue; + case '\t': append("\\t"); continue; + } + if (cp < 0x80) { hex(cp); continue; } + int continuation; + uint32_t minimum; + if (cp >= 0xc2 && cp <= 0xdf) { continuation = 1; minimum = 0x80; cp &= 31; } + else if (cp >= 0xe0 && cp <= 0xef) { continuation = 2; minimum = 0x800; cp &= 15; } + else if (cp >= 0xf0 && cp <= 0xf4) { continuation = 3; minimum = 0x10000; cp &= 7; } + else throw std::invalid_argument("Invalid UTF-8 in Arrow string"); + if (continuation > n - i) throw std::invalid_argument("Truncated UTF-8 in Arrow string"); + for (int j = 0; j < continuation; ++j) { + uint8_t byte = static_cast(s[i++]); + if ((byte & 0xc0) != 0x80) throw std::invalid_argument("Invalid UTF-8 continuation"); + cp = (cp << 6) | (byte & 63); + } + if (cp < minimum || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) + throw std::invalid_argument("Invalid Unicode code point in Arrow string"); + if (cp <= 0xffff) hex(cp); + else { cp -= 0x10000; hex(0xd800 + (cp >> 10)); hex(0xdc00 + (cp & 1023)); } + } + put('"'); + } + void temporal(int64_t value, bool date) { + int64_t seconds, micros = 0; + if (date) seconds = value * 86400; + else { + seconds = value / 1000000; + micros = value % 1000000; + if (micros < 0) { micros += 1000000; --seconds; } + } + time_t raw = static_cast(seconds); + struct tm parts {}; + if (static_cast(raw) != seconds || !gmtime_r(&raw, &parts)) + throw std::invalid_argument("Arrow timestamp outside platform calendar range"); + int64_t year = static_cast(parts.tm_year) + 1900; + put('"'); + if (year < 0) { put('-'); year = -year; } + if (year < 1000) put('0'); + if (year < 100) put('0'); + if (year < 10) put('0'); + integer(year); + char buf[32]; + int n = PyOS_snprintf(buf, sizeof(buf), "-%02d-%02d", parts.tm_mon + 1, parts.tm_mday); + append(buf, n); + if (!date) { + n = PyOS_snprintf(buf, sizeof(buf), "T%02d:%02d:%02d.%06dZ", + parts.tm_hour, parts.tm_min, parts.tm_sec, static_cast(micros)); + append(buf, n); + } + put('"'); + } +}; + +enum class Type { Int64, Double, Bool, String, Date, Timestamp }; +Type type_of(const ArrowSchema* s) { + if (!s || !s->format || s->dictionary || s->n_children) + throw std::invalid_argument("Native arrays encoder requires normalized scalar Arrow columns"); + const char* f = s->format; + if (!std::strcmp(f, "l")) return Type::Int64; + if (!std::strcmp(f, "g")) return Type::Double; + if (!std::strcmp(f, "b")) return Type::Bool; + if (!std::strcmp(f, "u")) return Type::String; + if (!std::strcmp(f, "tdD")) return Type::Date; + if (!std::strcmp(f, "tsu:UTC")) return Type::Timestamp; + throw std::invalid_argument(std::string("Unsupported native arrays Arrow format: ") + f); +} + +bool bit(const void* buffer, int64_t index) { + return (static_cast(buffer)[index >> 3] >> (index & 7)) & 1; +} +void cell(Writer& out, Type type, const ArrowArray* a, int64_t row) { + int64_t i = a->offset + row; + if (a->buffers[0] && !bit(a->buffers[0], i)) { out.append("null"); return; } + switch (type) { + case Type::Int64: { + int64_t v = static_cast(a->buffers[1])[i]; + bool quote = v < -9007199254740991LL || v > 9007199254740991LL; + if (quote) out.put('"'); + out.integer(v); + if (quote) out.put('"'); + break; + } + case Type::Double: { + double v = static_cast(a->buffers[1])[i]; + if (!std::isfinite(v)) throw std::invalid_argument("Out of range float values are not JSON compliant"); + char* text = PyOS_double_to_string(v, 'r', 0, Py_DTSF_ADD_DOT_0, nullptr); + if (!text) throw PythonError{}; + try { out.append(text); } catch (...) { PyMem_Free(text); throw; } + PyMem_Free(text); + break; + } + case Type::Bool: out.append(bit(a->buffers[1], i) ? "true" : "false"); break; + case Type::String: { + const auto* offsets = static_cast(a->buffers[1]); + int64_t start = offsets[i], end = offsets[i + 1]; + if (start < 0 || end < start) throw std::invalid_argument("Invalid Arrow string offsets"); + // Empty buffers may legally have a null data pointer. + const char* bytes = end == start ? "" : static_cast(a->buffers[2]) + start; + out.string(bytes, end - start); + break; + } + case Type::Date: out.temporal(static_cast(a->buffers[1])[i], true); break; + case Type::Timestamp: out.temporal(static_cast(a->buffers[1])[i], false); break; + } +} +struct SchemaOwner { + ArrowSchema value {}; + ~SchemaOwner() { if (value.release) value.release(&value); } +}; +struct ArrayOwner { + ArrowArray value {}; + ~ArrayOwner() { if (value.release) value.release(&value); } +}; +void stream_error(ArrowArrayStream* stream) { + const char* error = stream->get_last_error(stream); + throw std::invalid_argument(error ? error : "Arrow stream failed"); +} +PyObject* encode_rows(PyObject*, PyObject* args) { + PyObject* capsule; + Py_ssize_t limit; + if (!PyArg_ParseTuple(args, "On:encode_rows", &capsule, &limit)) return nullptr; + if (limit < 0) { PyErr_SetString(PyExc_ValueError, "Negative payload byte budget"); return nullptr; } + auto* stream = static_cast(PyCapsule_GetPointer(capsule, "arrow_array_stream")); + if (!stream) return nullptr; + if (!stream->release || !stream->get_schema || !stream->get_next || !stream->get_last_error) { + PyErr_SetString(PyExc_ValueError, "Released or invalid Arrow stream"); return nullptr; + } + // Borrow the stream for this synchronous call. Its PyCapsule owns/releases + // it; each schema and batch obtained here has its own release callback. + try { + SchemaOwner schema; + if (stream->get_schema(stream, &schema.value)) stream_error(stream); + const auto& s = schema.value; + if (!s.release || !s.format || std::strcmp(s.format, "+s") || s.dictionary || s.n_children < 0) + throw std::invalid_argument("Native arrays encoder requires an Arrow table stream"); + std::vector types; + for (int64_t j = 0; j < s.n_children; ++j) types.push_back(type_of(s.children[j])); + Writer out(static_cast(limit)); + out.put('['); + bool first = true; + while (true) { + ArrayOwner batch; + if (stream->get_next(stream, &batch.value)) stream_error(stream); + const auto& a = batch.value; + if (!a.release) break; + if (a.length < 0 || a.offset < 0 || a.n_children != s.n_children || a.null_count != 0) + throw std::invalid_argument("Invalid Arrow record batch"); + for (int64_t j = 0; j < a.n_children; ++j) { + const auto* child = a.children[j]; + int needed = types[j] == Type::String ? 3 : 2; + if (!child || child->offset < 0 || child->length < a.length + a.offset || + !child->buffers || child->n_buffers != needed) + throw std::invalid_argument("Invalid Arrow column layout"); + } + // Match zip(*columns) for the degenerate zero-column table. + for (int64_t row = 0; row < a.length && !types.empty(); ++row) { + if ((row & 4095) == 0 && PyErr_CheckSignals() < 0) throw PythonError{}; + if (!first) out.put(','); + first = false; + out.put('['); + for (int64_t j = 0; j < a.n_children; ++j) { + if (j) out.put(','); + cell(out, types[j], a.children[j], row + a.offset); + } + out.put(']'); + } + } + out.put(']'); + return PyBytes_FromStringAndSize(out.data.data(), static_cast(out.data.size())); + } catch (const PythonError&) { + if (!PyErr_Occurred()) PyErr_NoMemory(); + } catch (const std::bad_alloc&) { PyErr_NoMemory(); } + catch (const std::exception& e) { PyErr_SetString(PyExc_ValueError, e.what()); } + return nullptr; +} +PyMethodDef methods[] = {{"encode_rows", encode_rows, METH_VARARGS, + "Encode normalized Arrow C stream rows as compact ASCII JSON bytes."}, + {nullptr, nullptr, 0, nullptr}}; +PyModuleDef module = {PyModuleDef_HEAD_INIT, "_native_json", nullptr, -1, methods}; +} // namespace +PyMODINIT_FUNC PyInit__native_json() { return PyModule_Create(&module); } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_phase.py b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py new file mode 100644 index 000000000..5239a18b8 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small phase boundary for shared assembly code.""" +from contextlib import nullcontext + + +def phase(*_args, **_kwargs): + return nullcontext() diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py new file mode 100644 index 000000000..8d607386c --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Public kumo.pytorch columnar adapter backed by DiskGraph.""" +import hashlib +import importlib.metadata +import json +from contextlib import ExitStack +from pathlib import Path + +import duckdb +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from ._assembly import GraphAssembler +from ._contract import schema_for +from ._phase import phase +from ._validation import validate_contract +from .bundle import TYPES, quote +from .native_selection import inclusive_second + + +def compatibility_issues(workload): + """Report semantic gaps without importing optional native dependencies.""" + issues = [] + if workload.selection != "native_multihop": + issues.append("kumo-pytorch currently requires native_multihop selection") + for number, layer in enumerate(workload.layers): + active = [step for step in layer if step["fanout"] != 0] + if not active: + issues.append(f"layer {number}: kumo-pytorch cannot express an empty graph layer") + for step in active: + _, dest = workload.bundle.endpoint(step) + if not workload.bundle.tables[dest].get("time_column"): + issues.append(f"{step['relation']} {step['direction']}: uniform sampling into " + f"non-temporal table {dest} is unsupported by kumo-pytorch") + return list(dict.fromkeys(issues)) + + +def contract_for(workload): + issues = compatibility_issues(workload) + if issues: + raise ValueError("Cannot preserve FM sampling with kumo-pytorch:\n" + "\n".join(issues)) + from kumo.pytorch import api + bundle = workload.bundle + schema = api.DataSchema( + tables=tuple(api.TableSpec(name=name, primary_key=table["primary_key"], + columns={c: api.DType.from_arrow(TYPES[s["dtype"]]) + for c, s in table["columns"].items()}, + time_column=table.get("time_column") or api.NON_TEMPORAL) + for name, table in bundle.tables.items() if name in workload.table_names), + relations=tuple(api.RelationSpec(name=name, left_table=rel["child"], + left_key=rel["foreign_key"], right_table=rel["parent"], + right_key=bundle.tables[rel["parent"]]["primary_key"]) + for name, rel in bundle.relations.items() if name in workload.relation_names)) + layers = [] + for layer in workload.layers: + samples = [] + for step in layer: + if step["fanout"] == 0: + continue + _, dest = bundle.endpoint(step) + total = bundle.tables[dest]["rows"] + count = total if step["fanout"] is None else min(total, step["fanout"]) + if not count: + raise ValueError("kumo-pytorch migration cannot yet preserve empty destinations") + samples.append(api.GraphSample(relation=step["relation"], + direction=step["direction"], fanout=min(count, 1) + if step["direction"] == "forward" else count, strategy="recent")) + layers.append(api.GraphLayer(samples=tuple(samples))) + sample = api.SampleSpec(entity=api.EntitySpec(table=workload.entity), + neighborhoods={"fm": api.NeighborhoodSpec(layers=tuple(layers))}) + fields = {name: api.Fields(fields=tuple(api.Encode(field=column, encoding=api.Raw()) + for column in bundle.tables[name]["columns"])) for name in workload.table_names} + batch = api.BatchSpec(inputs={"graph": api.Input(source=api.NeighborhoodSource(name="fm"), + representation=api.Graph(node_fields=fields), layout=None)}, + batching=api.Batching(max_samples=workload.budgets["max_instances"])) + return api.DataLoaderSpec(schema=schema, sample=sample, batch=batch) + + +class KumoPytorchBackend: + """Use prepare → materialize(AnchorTable) → Task.batches, without RDL. + + Inherit only FM's retained columnar graph-to-payload assembly. No direct + SamplerEngine calls or private kumo.pytorch implementation hooks are used. + """ + + merged_edge_output = True + + def __init__(self, workload, index, *, threads=2): + self.output_format = "columnar" + contract = contract_for(workload) + from kumo.pytorch import api + validate_contract(workload) + self.bundle, self.index = workload.bundle, Path(index).resolve() + if self.index == self.bundle.root or self.bundle.root in self.index.parents: + raise ValueError("Native index must be outside the immutable bundle") + self.node_types = list(self.bundle.tables) + self.assembler = GraphAssembler(self.bundle) + self.metrics = {"kumo_pytorch_batches": 0, "multihop_selected_nodes": 0, + "multihop_selected_edges": 0, "native_candidate_rows": 0, + "api": "kumo.pytorch.Task.batches", "output_format": "inspectable"} + self.derived_source_bytes = 0 + self._stack, self._tasks = ExitStack(), {} + self._workload = workload + with duckdb.connect(config={"threads": threads}) as con: + for name in workload.table_names: + table = self.bundle.tables[name] + con.read_parquet(self.bundle.paths(name)).create_view("source") + for column, spec in table["columns"].items(): + if spec["dtype"] == "timestamp[us]": + fractional = con.execute(f"SELECT count(*) FROM source WHERE " + f"epoch_us({quote(column)}) % 1000000 != 0").fetchall()[0][0] + if fractional: + raise ValueError(f"kumo-pytorch requires whole-second timestamp features: {name}.{column}") + if table.get("time_column"): + nulls = con.execute(f"SELECT count(*) FROM source WHERE " + f"{quote(table['time_column'])} IS NULL").fetchall()[0][0] + if nulls: + raise ValueError(f"kumo-pytorch would drop NULL-clock source rows: {name}") + marker = self.index / "fm_snapshot.json" + if self.index.exists() and any(self.index.iterdir()) and not marker.exists(): + raise FileExistsError("Refusing an incomplete or unrelated kumo-pytorch index") + self.index.mkdir(parents=True, exist_ok=True) + provenance = {"snapshot_id": self.bundle.snapshot_id, "adapter_version": 1, + "contract": contract.fingerprint} + provenance["dependencies"] = { + name: {"version": (dist := importlib.metadata.distribution(name)).version, + "source": json.loads(dist.read_text("direct_url.json") or "null")} + for name in ("diskgraph", "kumo-pytorch")} + if marker.exists() and json.loads(marker.read_text()) != provenance: + raise ValueError("kumo-pytorch cache belongs to another snapshot/contract") + self.preparation_mode = "cached_index_open" if marker.exists() else "index_build" + try: + # Exact file inventories: do not glob sibling files into immutable data. + source_root = self.index / "sources" + sources = {} + for name in workload.table_names: + directory = source_root / name + directory.mkdir(parents=True, exist_ok=True) + for i, path in enumerate(self.bundle.paths(name)): + link = directory / f"{i:08d}.parquet" + if not link.exists(): + link.symlink_to(path) + sources[name] = api.Parquet(path=str(directory)) + with phase("kumo_pytorch_prepare", "preparation"): + self.plan = self._stack.enter_context(api.prepare( + contract=contract, dataset=api.Dataset(tables=sources), + engine=api.DiskGraphEngine(index=self.index / "engine", num_threads=threads))) + metadata = json.loads((self.index / "engine" / "index_metadata.json").read_text()) + counts = {t["name"]: t["max_id"] + 1 for t in metadata["node_types"]} + for name in workload.table_names: + if counts.get(name, 0) != self.bundle.tables[name]["rows"]: + raise ValueError(f"kumo-pytorch ingestion lost source rows: {name}") + marker.write_text(json.dumps(provenance, indent=2) + "\n") + except BaseException: + self.close() + raise + + @staticmethod + def _key(seeds): + return tuple((s["entity_key"], s["anchor_time"]) for s in seeds) + + def prepare_batch(self, seeds): + """Materialize this exact ordered request outside request timing.""" + from kumo.pytorch import api + key = self._key(seeds) + if key in self._tasks: + return + token = hashlib.sha256(repr(key).encode()).hexdigest() + path = self.index / f"anchors-{token}.parquet" + pk = self.bundle.tables[self._workload.entity]["primary_key"] + dtype = TYPES[self.bundle.tables[self._workload.entity]["columns"][pk]["dtype"]] + # Graph neighborhoods use the native inclusive whole-second cutoff. + anchors = pa.table({"entity": pa.array([s["entity_key"] for s in seeds], dtype), + "sample": pa.array(range(len(seeds)), pa.int64()), + "at": pa.array([inclusive_second(s["anchor_time"]) for s in seeds], + pa.timestamp("s", tz="UTC"))}) + pq.write_table(anchors, path) + with phase("kumo_pytorch_materialize", "preparation"): + task = self._stack.enter_context(self.plan.materialize(api.AnchorTable( + source=api.Parquet(path=str(path)), entity_id="entity", sample_id="sample", + as_of_time="at"))) + if task.count() != len(seeds): + raise ValueError("kumo-pytorch materialization lost anchors") + self._tasks[key] = task + + def extract(self, workload, seeds): + if not seeds: + return {name: pa.table({column: pa.array([], type=TYPES[spec['dtype']]) + for column, spec in table['columns'].items()}) + for name, table in schema_for(workload)['related_tables'].items()} + if workload.id != self._workload.id: + raise ValueError("kumo-pytorch backend is bound to its prepared workload") + key = self._key(seeds) + if key not in self._tasks: + raise ValueError("Call prepare_batch(seeds) before measuring a new request") + if (len(seeds) > workload.budgets["max_instances"] or + len(seeds) > workload.budgets["max_rows_per_table"]): + raise ValueError("Native-multihop root budget exceeded") + with phase("kumo_pytorch_batches", "retrieval_selection"): + batches = iter(self._tasks[key].batches(batch_size=len(seeds), shuffle=False, + output_format=self.output_format)) + batch = next(batches) + if next(batches, None) is not None: + raise ValueError("Expected exactly one native anchor batch") + self.metrics["kumo_pytorch_batches"] += 1 + with phase("kumo_pytorch_graph_adaptation", "graph_assembly"): + return self._adapt_columnar_batch(workload, seeds, batch) + + def _adapt_columnar_batch(self, workload, seeds, batch): + """Consume public Arrow columns; only reorder when anchor order requires it.""" + from kumo.pytorch.api import ColumnarGraph + graph = batch.inputs["graph"] + if not isinstance(graph, ColumnarGraph) or graph.root_type != workload.entity: + raise ValueError("Expected a columnar graph rooted at the workload entity") + permutation = np.asarray(batch.metadata["sample_ids"], dtype=np.int64) + if not np.array_equal(np.sort(permutation), np.arange(len(seeds))): + raise ValueError("kumo-pytorch anchor IDs are not a permutation of the request") + same_order = np.array_equal(permutation, np.arange(len(seeds))) + nodes, features, edges, directions, inverse = {}, {}, {}, {}, {} + for name, node in graph.nodes.items(): + type_id = self.node_types.index(name) + positions = node.owners.to_numpy(zero_copy_only=False) + if (node.owners.null_count or np.any(positions >= len(seeds)) or + np.any(positions < 0)): + raise ValueError("kumo-pytorch node owner is out of bounds") + table = node.features + if len(table) != len(node.ids) or len(table) != len(positions): + raise ValueError("kumo-pytorch columnar node/feature alignment mismatch") + owners = node.owners.cast(pa.uint64()) if same_order else pa.array(permutation[positions], pa.uint64()) + order = None + if not same_order and name == workload.entity: + order = np.arange(len(positions)) + order[:len(seeds)] = np.argsort(permutation[positions[:len(seeds)]]) + undo = np.empty(len(order), dtype=np.int64) + undo[order] = np.arange(len(order)) + inverse[name] = undo + owners, table = owners.take(pa.array(order)), table.take(pa.array(order)) + nodes[type_id] = [pa.record_batch( + [owners, pa.array(np.arange(len(positions)), pa.uint64())], names=["id_a", "id_b"])] + features[type_id] = table.to_batches() or [pa.RecordBatch.from_arrays( + [table[c].combine_chunks() for c in table.column_names], schema=table.schema)] + for rel_id, (name, edge) in enumerate(graph.edges.items()): + rel = self.bundle.relations[name] + orientation = (edge.source_type, edge.destination_type) + if orientation not in ((rel["child"], rel["parent"]), + (rel["parent"], rel["child"])): + raise ValueError("kumo-pytorch relation orientation differs from the schema") + native_types = orientation + columns = [] + for column, table_name in zip(("source", "destination"), native_types, strict=True): + values = edge.index[column] + if table_name in inverse: + indices = values.to_numpy(zero_copy_only=False) + if values.null_count or np.any(indices < 0) or np.any(indices >= len(inverse[table_name])): + raise ValueError("kumo-pytorch edge endpoint is out of bounds") + values = pa.chunked_array([pa.array(inverse[table_name][indices], pa.uint64())]) + columns.append(values.cast(pa.uint64())) + if orientation != (rel["child"], rel["parent"]): + columns.reverse() + index = pa.Table.from_arrays(columns, names=["id_a", "id_b"]) + directions[rel_id] = name, 'forward', rel["child"], rel["parent"] + edges[rel_id] = index.to_batches() or [pa.RecordBatch.from_arrays( + [index[c].combine_chunks() for c in index.column_names], schema=index.schema)] + count = sum(len(b) for group in (*nodes.values(), *edges.values()) for b in group) + if count > workload.budgets["max_candidate_rows"]: + raise ValueError("Native-multihop node-plus-edge budget exceeded") + self.metrics["native_candidate_rows"] += count + return self.assembler._adapt(workload, seeds, np.arange(len(seeds), dtype=np.uint64), + nodes, features, edges, directions) + + def close(self): + self._stack.close() diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py new file mode 100644 index 000000000..ec1f67f0f --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py @@ -0,0 +1,395 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""FM orchestration over an already indexed public PyTorch engine.""" +from __future__ import annotations + +import tempfile +import uuid +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace + +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from ._assembly import GraphAssembler +from ._pytorch import KumoPytorchBackend +from .batch import PayloadBatch +from .bundle import TYPES +from .workload import timestamp, validate_scalar + + +@dataclass(frozen=True) +class Context: + """Historical examples: entities × times, or ordered explicit pairs. + + A pair may include a third, explicit int64 occurrence ID. Repeated pairs + remain distinct; omitted IDs are their zero-based positions in the split. + """ + + entities: object = () + times: object = () + pairs: object = () + + +@dataclass(frozen=True) +class Query: + """Prediction examples: entities at one time, or ordered explicit pairs.""" + + entities: object = () + at: object = None + pairs: object = () + + +def _rows(population, split, key_dtype): + if split == "context" and not isinstance(population, Context): + raise TypeError("context requires fm.Context") + if split == "predict" and not isinstance(population, Query): + raise TypeError("query requires fm.Query") + pairs = list(population.pairs) + entities = list(population.entities) + if pairs and (entities or (split == "context" and population.times) or + (split == "predict" and population.at is not None)): + raise ValueError("Choose explicit pairs or entities with times") + if not pairs: + if split == "context": + pairs = [(entity, at) for at in population.times for entity in entities] + else: + pairs = [(entity, population.at) for entity in entities] + if not pairs: + raise ValueError(f"{split} needs at least one occurrence") + result = [] + seen = set() + for position, item in enumerate(pairs): + if len(item) not in (2, 3): + raise ValueError("Occurrences require (entity, time[, instance_id])") + entity, at = item[:2] + validate_scalar(entity, key_dtype) + instant = timestamp(at) + identifier = item[2] if len(item) == 3 else position + validate_scalar(identifier, "int64") + if identifier in seen: + raise ValueError("Duplicate occurrence ID within a split") + seen.add(identifier) + result.append({"instance_id": identifier, "entity_key": entity, + "anchor_time": instant}) + return result + + +def _schema_view(schema): + """Derive FM's supported scalar schema without touching the base index.""" + from kumo.pytorch.contract.schema import table_clock + + tables = {} + for table in schema.tables: + if table.primary_key is None: + raise ValueError(f"FM requires a primary key: {table.name}") + columns = {} + for name, dtype in table.columns.items(): + arrow = dtype.to_arrow() + if pa.types.is_timestamp(arrow) and arrow.unit != "ns": + physical = "timestamp[us]" + elif pa.types.is_date32(arrow): + physical = "date" + elif pa.types.is_float64(arrow): + physical = "float64" + elif pa.types.is_boolean(arrow): + physical = "bool" + elif pa.types.is_int64(arrow): + physical = "int64" + elif pa.types.is_string(arrow): + physical = "string" + else: + physical = str(arrow) + if physical not in TYPES: + raise ValueError(f"FM cannot encode {table.name}.{name}: {arrow}") + columns[name] = {"dtype": physical, + "nullable": name != table.primary_key} + if columns[table.primary_key]["dtype"] not in ("int64", "string"): + raise ValueError(f"FM needs an int64/string key: {table.name}") + tables[table.name] = {"primary_key": table.primary_key, + "time_column": table_clock(table), "columns": columns} + relations = {} + for relation in schema.relations: + left, right = tables[relation.left_table], tables[relation.right_table] + if relation.right_key == right["primary_key"]: + child, parent, fk = relation.left_table, relation.right_table, relation.left_key + elif relation.left_key == left["primary_key"]: + child, parent, fk = relation.right_table, relation.left_table, relation.right_key + else: + raise ValueError(f"FM requires one primary-key side: {relation.name}") + relations[relation.name] = {"child": child, "parent": parent, "foreign_key": fk} + return SimpleNamespace(tables=tables, relations=relations) + + +def _target_horizon(value): + from kumo.pytorch import api + + aggregate = value.left if isinstance(value, api.Compare) else value + if not isinstance(aggregate, api.Aggregate) or not isinstance(aggregate.source, api.Rows): + raise ValueError("FM targets require a bounded Rows aggregate or comparison") + window = aggregate.source.time + if window is None or window.start != 0 or window.stop is None or window.stop <= 0: + raise ValueError("FM target needs a future window (0, positive stop]") + return window.stop + + +def _selected_graph(schema, sample, features, bundle): + from kumo.pytorch import api + + if len(sample.neighborhoods) != 1: + raise ValueError("FM requires exactly one declared NeighborhoodSpec") + graph_name, neighborhood = next(iter(sample.neighborhoods.items())) + frontier = {sample.entity.table} + tables, relations = set(frontier), set() + if not neighborhood.layers: + raise ValueError("FM requires at least one graph layer") + for layer in neighborhood.layers: + if not layer.samples: + raise ValueError("FM cannot sample an empty graph layer") + following = set() + for step in layer.samples: + if step.strategy != "recent" or not step.fanout or step.fanout < 0: + raise ValueError("FM supports bounded recent graph sampling only") + if step.relation not in bundle.relations: + raise ValueError(f"Unknown relation: {step.relation}") + rel = bundle.relations[step.relation] + source, dest = ((rel["child"], rel["parent"]) if step.direction == "forward" + else (rel["parent"], rel["child"])) + if source not in frontier: + raise ValueError(f"Unreachable graph layer: {source}") + if bundle.tables[dest]["time_column"] is None: + raise ValueError(f"FM cannot sample uniformly into timeless table {dest}") + following.add(dest) + tables.update((source, dest)) + relations.add(step.relation) + frontier = following + if set(features) != tables: + raise ValueError(f"features must name reachable tables: {sorted(tables)}") + projections = {} + fields = {} + for name in sorted(tables): + declared = bundle.tables[name]["columns"] + requested = list(features[name]) + if len(requested) != len(set(requested)) or not set(requested) <= set(declared): + raise ValueError(f"Invalid FM feature projection: {name}") + pk = bundle.tables[name]["primary_key"] + projections[name] = list(dict.fromkeys((pk, *requested))) + internal = list(projections[name]) + clock = bundle.tables[name]["time_column"] + if clock: + internal.append(clock) + internal.extend(bundle.relations[r]["foreign_key"] for r in relations + if bundle.relations[r]["child"] == name) + fields[name] = api.Fields(tuple(api.Encode(column, api.Raw()) + for column in dict.fromkeys(internal))) + return graph_name, fields, tables, relations, projections + + +class SnapshotFMLoader: + """Prepare FM labels and columnar graph plans over one adopted snapshot.""" + + def __init__(self, *, engine, sample, target, task, features, classes=None, + batch_size=1024, budgets=None, schema=None): + from kumo.pytorch import api + + from .loader import PreparedRequest + + if not isinstance(engine, api.DiskGraphEngine) or not isinstance(sample, api.SampleSpec): + raise TypeError("Provide an adopted DiskGraphEngine and SampleSpec") + schema = engine.schema if schema is None else schema + if not isinstance(schema, api.DataSchema): + raise TypeError("schema must be a public DataSchema projection or alias") + if target not in sample.targets or task not in ("regression", "classification", "binary_classification"): + raise ValueError("Choose a declared target and supported FM task") + self._prepared_request_type = PreparedRequest + self._closed = False + self._engine, self._target = engine, target + self._bundle = _schema_view(schema) + self._entity = sample.entity.table + self._key_type = self._bundle.tables[self._entity]["columns"][ + self._bundle.tables[self._entity]["primary_key"]]["dtype"] + self._horizon = _target_horizon(sample.targets[target]) + self._kind = "regression" if task == "regression" else "binary_classification" + if self._kind == "regression": + if classes is not None: + raise ValueError("Regression has no classes") + self._classes = None + target_spec = {"dtype": "float64"} + else: + if not isinstance(classes, dict) or len(classes) != 2 or len(set(classes.values())) != 2: + raise ValueError("Binary classification requires a two-entry raw-value to class-name map") + self._classes = dict(classes) + target_spec = {"dtype": "string", "classes": list(classes.values())} + graph_name, fields, tables, relations, projections = _selected_graph( + schema, sample, features, self._bundle) + if not isinstance(batch_size, int) or batch_size < 1: + raise ValueError("batch_size must be positive") + default_budgets = {"max_instances": 10000, "max_rows_per_table": 1000000, + "max_payload_bytes": 30 * 1024**2, + "max_decoded_arrow_bytes": 30 * 1024**2, + "max_columns": 500, "max_candidate_rows": 1000000} + default_budgets.update(budgets or {}) + if any(type(value) is not int or value < 1 for value in default_budgets.values()): + raise ValueError("FM budgets must be positive integers") + self._batch_size = min(batch_size, default_budgets["max_instances"]) + self.workload = SimpleNamespace(bundle=self._bundle, entity=self._entity, + kind=self._kind, target=target_spec, table_names=tables, + relation_names=relations, projections=projections, budgets=default_budgets) + self._adapter = SimpleNamespace(bundle=self._bundle, + node_types=list(self._bundle.tables), assembler=GraphAssembler(self._bundle), + metrics={"native_candidate_rows": 0}) + self._stack = ExitStack() + self._temporary = tempfile.TemporaryDirectory(prefix="kumo-fm-anchors-") + pk = self._bundle.tables[self._entity]["primary_key"] + label_sample = api.SampleSpec(entity=sample.entity, + entity_values={pk: api.Field(pk)}, targets={target: sample.targets[target]}) + label_batch = api.BatchSpec(inputs={"entity": api.Input( + api.EntitySource(), api.Fields((api.Encode(pk, api.Raw()),)), api.Separate())}, + targets={"label": api.LabelTarget(target, api.Raw())}, + batching=api.Batching(max_samples=self._batch_size)) + graph_sample = api.SampleSpec(entity=sample.entity, + neighborhoods={graph_name: sample.neighborhoods[graph_name]}) + graph_batch = api.BatchSpec(inputs={"graph": api.Input( + api.NeighborhoodSource(graph_name), api.Graph(node_fields=fields), None)}, + batching=api.Batching(max_samples=self._batch_size)) + try: + self._labels = self._stack.enter_context(api.prepare( + contract=api.DataLoaderSpec(schema, label_sample, label_batch), engine=engine)) + self._graphs = self._stack.enter_context(api.prepare( + contract=api.DataLoaderSpec(schema, graph_sample, graph_batch), engine=engine)) + except BaseException: + self.close() + raise + + @property + def preparation_mode(self): + return "existing_snapshot" + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + if not self._closed: + self._closed = True + self._stack.close() + self._temporary.cleanup() + + def materialize(self, *, context=None, query=None): + if self._closed: + raise RuntimeError("Loader is closed") + if context is None and query is None: + raise ValueError("Provide context or query occurrences") + splits = {} + for split, population in (("context", context), ("predict", query)): + if population is not None: + seeds = _rows(population, split, self._key_type) + if len(seeds) > self.workload.budgets["max_instances"]: + raise ValueError("FM instance budget exceeded") + splits[split] = seeds + if "context" in splits: + if "predict" not in splits: + raise ValueError("Context labels require a query cutoff") + cutoff = min(row["anchor_time"] for row in splits["predict"]) + if any(row["anchor_time"] + timedelta(seconds=self._horizon) > cutoff + for row in splits["context"]): + raise ValueError("Context label window is not observable before the earliest query") + return self._prepared_request_type(self, splits) + + def load(self, *, context=None, query=None): + with self.materialize(context=context, query=query) as request: + return request.load() + + def _anchor(self, seeds): + from kumo.pytorch import api + + path = Path(self._temporary.name) / f"{uuid.uuid4().hex}.parquet" + pq.write_table(pa.table({ + "entity": pa.array([s["entity_key"] for s in seeds], TYPES[self._key_type]), + "sample": pa.array([s["instance_id"] for s in seeds], pa.int64()), + "at": pa.array([s["anchor_time"] for s in seeds], pa.timestamp("us", tz="UTC")), + }), path) + return api.AnchorTable(api.Parquet(path=str(path)), entity_id="entity", + sample_id="sample", as_of_time="at") + + def _labels_for(self, anchor, seeds): + result = {} + expected = {s["instance_id"] for s in seeds} + with self._labels.materialize(anchor) as task: + for batch in task.batches(output_format="inspectable", shuffle=False, + batch_size=self._batch_size): + label = batch.targets["label"] + sample_ids = list(batch.metadata["sample_ids"]) + values, valid = label["values"].tolist(), label["valid"].tolist() + if len(sample_ids) != len(values) or len(values) != len(valid): + raise ValueError("FM label batch is misaligned") + for identifier, value, present in zip(sample_ids, values, valid, strict=True): + identifier = int(identifier) + if identifier not in expected or identifier in result or not present: + raise ValueError("Missing, duplicated, or unexpected FM context label") + if self._classes is not None: + if value not in self._classes: + raise ValueError("Context label is outside declared classes") + value = self._classes[value] + else: + value = float(value) + result[identifier] = value + if set(result) != expected: + raise ValueError("Missing FM context labels") + return result + + def _graphs_for(self, anchor, seeds): + from ._contract import schema_for + + expected = {s["instance_id"]: s for s in seeds} + seen = set() + parts = {name: [] for name in schema_for(self.workload)["related_tables"]} + with self._graphs.materialize(anchor) as task: + for batch in task.batches(output_format="columnar", shuffle=False, + batch_size=self._batch_size): + identifiers = [int(value) for value in batch.metadata["sample_ids"]] + if (len(identifiers) != len(set(identifiers)) or + any(value not in expected or value in seen for value in identifiers)): + raise ValueError("Duplicate or unexpected FM graph occurrence") + seen.update(identifiers) + chunk = [expected[value] for value in identifiers] + local = SimpleNamespace(inputs=batch.inputs, + metadata={"sample_ids": list(range(len(chunk)))}) + related = KumoPytorchBackend._adapt_columnar_batch( + self._adapter, self.workload, chunk, local) + for name, table in related.items(): + parts[name].append(table) + if seen != set(expected): + raise ValueError("Missing FM graph occurrences") + order = {seed["instance_id"]: i for i, seed in enumerate(seeds)} + result = {} + for name, chunks in parts.items(): + table = pa.concat_tables(chunks) if chunks else pa.table({}) + if len(table): + rank = pa.array([order[int(value)] for value in table["instance_id"].to_pylist()]) + index = pc.sort_indices(pa.table({"rank": rank, + "key": table[self._bundle.tables[name]["primary_key"]]}), + sort_keys=[("rank", "ascending"), ("key", "ascending")]) + table = table.take(index) + result[name] = table + return result + + def _load(self, splits): + if self._closed: + raise RuntimeError("Loader is closed") + ready = {} + for split, seeds in splits.items(): + anchor = self._anchor(seeds) + if split == "context": + labels = self._labels_for(anchor, seeds) + seeds = [dict(seed, target=labels[seed["instance_id"]]) for seed in seeds] + ready[split] = seeds, self._graphs_for(anchor, seeds) + return PayloadBatch(self.workload, ready) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_validation.py b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py new file mode 100644 index 000000000..b79a61781 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Preparation-time native sampling compatibility checks.""" +import duckdb + +from ._phase import phase +from .bundle import quote + + +def validate_contract(workload): + """Preparation-time eligibility gate shared by all engines; no hidden fallback. + + Null sort keys are admitted by the upstream sampler, and subsecond clocks + cannot be indexed exactly. Reject those destinations before any measurement. + Fractional non-clock features remain supported by the existing ingest bridge. + """ + destinations = {workload.bundle.endpoint(s)[1] for layer in workload.layers for s in layer + if s['fanout'] != 0} + bundle = workload.bundle + with phase('multihop_contract_validation', 'preparation'): + with duckdb.connect(config={'threads': 2, 'memory_limit': '1GB'}) as con: + for name in sorted(destinations): + table = bundle.tables[name] + tc = table.get('time_column') + if not tc or not table['rows']: + continue + con.read_parquet(bundle.paths(name)).create_view('clock_source') + nulls, fractional, low, high = con.execute( + f'SELECT count(*) - count({quote(tc)}), ' + f'count(*) FILTER (WHERE epoch_us({quote(tc)}) % 1000000 != 0), ' + f'min(epoch_us({quote(tc)})), max(epoch_us({quote(tc)})) FROM clock_source' + ).fetchall()[0] + if nulls or fractional or (high - low) // 1000000 > 2 * (2**31 - 1): + raise ValueError(f'native_multihop requires non-null, whole-second, indexable clocks: {name}.{tc}; ' + 'use temporal_query or local_native for exact fallbacks') + for layer in workload.layers: + for step in layer: + total = bundle.tables[bundle.endpoint(step)[1]]['rows'] + fanout = total if step['fanout'] is None else min(total, step['fanout']) + if step['direction'] == 'forward': + fanout = min(fanout, 1) + if fanout > 2**32 - 1: + raise ValueError('native_multihop fanout exceeds uint32; use a bounded fanout') diff --git a/packages/experimental/kumo-fm/src/kumo/fm/batch.py b/packages/experimental/kumo-fm/src/kumo/fm/batch.py new file mode 100644 index 000000000..07056e097 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/batch.py @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Retained FM Arrow tables and explicit HTTP encoding.""" +from __future__ import annotations + +from copy import deepcopy +from types import MappingProxyType + +import pyarrow as pa + +from . import native_json +from ._contract import metadata_for +from ._encoding import encode_arrow_table +from .bundle import TYPES + + +def _typed_table(table, columns, max_columns): + if len(columns) > max_columns: + raise ValueError("Payload table exceeds column budget") + schema = pa.schema([pa.field(name, TYPES[spec["dtype"]], nullable=spec["nullable"]) + for name, spec in columns.items()]) + table = table.select(list(columns)).cast(schema) + for field in schema: + if not field.nullable and table[field.name].null_count: + raise ValueError(f"Null in required Arrow field: {field.name}") + return table + + +class PayloadBatch: + """One sampled request; Arrow tables stay valid after loader closure.""" + + def __init__(self, workload, splits): + metadata = metadata_for(workload) + self._metadata = deepcopy(metadata) + self._budgets = dict(workload.budgets) + self._splits = {} + logical_bytes = 0 + for split, (seeds, related) in splits.items(): + columns = dict(metadata["schema"]["instance_table"]["columns"]) + if split == "predict": + columns.pop("__fm_target") + instance = pa.table({ + name: pa.array([str(seed["target"]) if name == "__fm_target" + and workload.kind != "regression" else seed[{ + "instance_id": "instance_id", + "__fm_entity_key": "entity_key", + "__fm_anchor": "anchor_time", + "__fm_target": "target", + }[name]] for seed in seeds], type=TYPES[spec["dtype"]]) + for name, spec in columns.items() + }) + instance = _typed_table(instance, columns, self._budgets["max_columns"]) + if len(instance) > self._budgets["max_rows_per_table"]: + raise ValueError("Instance table exceeds row budget") + normalized = {} + for name, spec in metadata["schema"]["related_tables"].items(): + table = _typed_table(related[name], spec["columns"], self._budgets["max_columns"]) + if len(table) > self._budgets["max_rows_per_table"]: + raise ValueError("Related table exceeds row budget") + normalized[name] = table + logical_bytes += table.nbytes + logical_bytes += instance.nbytes + self._splits[split] = (instance, MappingProxyType(normalized)) + if logical_bytes > self._budgets["max_decoded_arrow_bytes"]: + raise ValueError("Decoded Arrow budget exceeded") + self._logical_bytes = logical_bytes + + @property + def metadata(self): + return deepcopy(self._metadata) + + @property + def logical_arrow_bytes(self): + return self._logical_bytes + + @property + def splits(self): + return tuple(self._splits) + + def tables(self, split): + instance, related = self._splits[split] + return MappingProxyType({"instance_table": instance, "related_tables": related}) + + def as_arrow_request(self): + result = self.metadata + for split, (instance, related) in self._splits.items(): + result[split] = {"instance_table": instance, "related_tables": dict(related)} + return result + + def encode(self, encoding="arrays"): + if encoding not in ("arrays", "arrow_ipc"): + raise ValueError("Unknown encoding") + payload = self.metadata + for split, (instance, related) in self._splits.items(): + instance_columns = dict(payload["schema"]["instance_table"]["columns"]) + if split == "predict": + instance_columns.pop("__fm_target") + encoded_instance, _ = encode_arrow_table( + instance, instance_columns, encoding, self._budgets, native_arrays=encoding == "arrays") + encoded_related = {} + for name, table in related.items(): + encoded_related[name], _ = encode_arrow_table( + table, payload["schema"]["related_tables"][name]["columns"], encoding, + self._budgets, native_arrays=encoding == "arrays") + payload[split] = {"instance_table": encoded_instance, "related_tables": encoded_related} + if encoding == "arrays": + return native_json.dumps(payload, self._budgets["max_payload_bytes"]) + import json + body = json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() + if len(body) > self._budgets["max_payload_bytes"]: + raise ValueError("Payload exceeds byte budget; reduce the workload batch") + return body diff --git a/packages/experimental/kumo-fm/src/kumo/fm/bundle.py b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py new file mode 100644 index 000000000..270203af7 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validated Parquet bundle metadata, shared by every backend. + +Full validation uses streaming file hashes and spillable DuckDB constraints; +only task rows and sampled graph rows need to become Python objects. +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +TYPES = {"int64": pa.int64(), "float64": pa.float64(), "bool": pa.bool_(), + "string": pa.string(), "date": pa.date32(), "timestamp[us]": pa.timestamp("us", tz="UTC")} + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def identity(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def check_name(name: str) -> None: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) or name.startswith("__") or name == "instance_id": + raise ValueError(f"Unsupported or reserved schema name: {name!r}") + + +class Bundle: + def __init__(self, root: Path, *, verify_files: bool = True): + self.root = Path(root).resolve() + self.manifest = json.loads((self.root / "manifest.json").read_text()) + m = self.manifest + self.dataset_manifest = m if m.get("format") == "fm-dataset-v1" else None + if self.dataset_manifest is None and m.get("format_version") != 2: + raise ValueError("Expected dataset-suite bundle format_version=2") + complete = json.loads((self.root / "_COMPLETE.json").read_text()) + if complete.get("manifest_sha256") != digest(self.root / "manifest.json"): + raise ValueError("Incomplete or changed bundle manifest") + validation = json.loads((self.root / "validation.json").read_text()) + if (validation.get("status") != ("pass" if self.dataset_manifest else "passed") or + complete.get("validation_sha256") != digest(self.root / "validation.json")): + raise ValueError("Bundle has no matching completed validation") + if self.dataset_manifest is not None: + from .datasets import manifest_view + self.manifest = m = manifest_view(self.dataset_manifest, self.root) + self.tables = m["tables"] + self.relations = m["relations"] + self.logical_id = m["logical_id"] + self.snapshot_id = self.dataset_manifest["snapshot_id"] if self.dataset_manifest else identity(m) + if not self.tables: + raise ValueError("Bundle must declare tables") + for name, table in self.tables.items(): + check_name(name) + columns = table["columns"] + for column, spec in columns.items(): + check_name(column) + if spec["dtype"] not in TYPES or type(spec.get("nullable")) is not bool: + raise ValueError(f"Unsupported column specification: {name}.{column}") + pk = table["primary_key"] + if pk not in columns or columns[pk]["dtype"] not in ("int64", "string") or columns[pk]["nullable"]: + raise ValueError(f"Table {name} needs a non-null int64/string primary key") + tc = table.get("time_column") + if tc is not None and (tc not in columns or columns[tc]["dtype"] not in ("timestamp[us]", "date")): + raise ValueError(f"Invalid time column: {name}") + if tc is None and table.get("availability") != "timeless": + raise ValueError(f"Declare {name} temporal or explicitly timeless") + if type(table["rows"]) is not int or table["rows"] < 0 or not table["files"]: + raise ValueError(f"Invalid table inventory: {name}") + if table["files"] != sorted(set(table["files"])): + raise ValueError(f"Table file inventory must be sorted and unique: {name}") + if any(p not in m["files"] for p in table["files"]): + raise ValueError(f"Uninventoried table file: {name}") + for name, rel in self.relations.items(): + check_name(name) + child, parent = self.tables[rel["child"]], self.tables[rel["parent"]] + fk = child["columns"][rel["foreign_key"]] + pk = parent["columns"][parent["primary_key"]] + if fk["dtype"] != pk["dtype"]: + raise ValueError(f"Incompatible relation key types: {name}") + if verify_files: + for name, entry in m["files"].items(): + if digest(self.path(name)) != entry["sha256"]: + raise ValueError(f"Bundle checksum mismatch: {name}") + + def path(self, relative: str) -> Path: + path = (self.root / relative).resolve() + if self.root not in path.parents: + raise ValueError("Bundle file path escapes the dataset directory") + return path + + def paths(self, name: str) -> list[str]: + return [str(self.path(p)) for p in self.tables[name]["files"]] + + def schema(self, name: str) -> pa.Schema: + return pa.schema([pa.field(c, TYPES[s["dtype"]], s["nullable"]) + for c, s in self.tables[name]["columns"].items()]) + + def endpoint(self, step: dict) -> tuple[str, str]: + rel = self.relations[step["relation"]] + return (rel["child"], rel["parent"]) if step["direction"] == "forward" else (rel["parent"], rel["child"]) + + def register(self, con, *, native: bool = False) -> None: + for name in self.tables: + con.read_parquet(self.paths(name)).create_view("__source_" + name) + kind = "TABLE" if native else "VIEW" + con.execute(f"CREATE {kind} {quote(name)} AS SELECT * FROM {quote('__source_' + name)}").fetchall() + if native: + con.execute("ANALYZE").fetchall() + + def validate(self, *, memory_limit: str = "1GB", threads: int = 2) -> dict: + """Full source validation; preparation cost, never timed as retrieval.""" + if getattr(self, "dataset_manifest", None) is not None: + from ._dataset_support.validate import validate + validate(self.root) + result = {"tables": {}, "relations": {}} + with duckdb.connect(config={"memory_limit": memory_limit, "threads": threads}) as con: + self.register(con) + for name, table in self.tables.items(): + expected = self.schema(name) + for path in self.paths(name): + parquet = pq.ParquetFile(path) + if not parquet.schema_arrow.equals(expected): + raise ValueError(f"Parquet schema differs from manifest: {path}") + entry = self.manifest["files"][str(Path(path).relative_to(self.root))] + if parquet.metadata.num_rows != entry["rows"]: + raise ValueError(f"File row count differs from inventory: {path}") + n = con.execute(f"SELECT count(*) FROM {quote(name)}").fetchall()[0][0] + if n != table["rows"]: + raise ValueError(f"Table row count mismatch: {name}") + for col, spec in table["columns"].items(): + if not spec["nullable"] and con.execute( + f"SELECT count(*) FROM {quote(name)} WHERE {quote(col)} IS NULL" + ).fetchall()[0][0]: + raise ValueError(f"Null required column: {name}.{col}") + if spec["dtype"] == "float64" and con.execute( + f"SELECT count(*) FROM {quote(name)} WHERE NOT isfinite({quote(col)})" + ).fetchall()[0][0]: + raise ValueError(f"Nonfinite feature: {name}.{col}") + pk = quote(table["primary_key"]) + if con.execute(f"SELECT count(*) FROM (SELECT {pk} FROM {quote(name)} GROUP BY {pk} HAVING count(*) > 1)").fetchall()[0][0]: + raise ValueError(f"Duplicate primary key: {name}") + result["tables"][name] = {"rows": n} + for name, rel in self.relations.items(): + pk = self.tables[rel["parent"]]["primary_key"] + child, parent, fk = quote(rel["child"]), quote(rel["parent"]), quote(rel["foreign_key"]) + bad = con.execute(f"SELECT count(*) FROM {child} c LEFT JOIN {parent} p ON c.{fk}=p.{quote(pk)} WHERE c.{fk} IS NOT NULL AND p.{quote(pk)} IS NULL").fetchall()[0][0] + if bad: + raise ValueError(f"Dangling foreign keys: {name} ({bad})") + nulls = con.execute(f"SELECT count(*) FROM {child} WHERE {fk} IS NULL").fetchall()[0][0] + result["relations"][name] = {"null_links": nulls, "edges": self.tables[rel["child"]]["rows"] - nulls} + self.validation = result + return result diff --git a/packages/experimental/kumo-fm/src/kumo/fm/datasets.py b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py new file mode 100644 index 000000000..e23e897fa --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py @@ -0,0 +1,103 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Adapt fm_datasets' published bundles without rewriting their source artifacts.""" +from copy import deepcopy + +from ._dataset_support.config import Config +from ._dataset_support.generate import DAY_US, schema_for +from ._dataset_support.storage import canonical_hash, safe_path + + +def manifest_view(source, root): + """Expose the loader's internal schema while retaining producer identities.""" + cfg = Config(**source["recipe"]["config"]) + tables, relations, files, tasks = {}, {}, {}, {} + for name, recipe in source["recipe"]["tables"].items(): + schema = schema_for(recipe, cfg) + columns = {} + for col, field in zip(recipe["columns"], schema, strict=True): + spec = {"dtype": "timestamp[us]" if col["type"] == "time" else col["type"], + "nullable": field.nullable} + if col["role"] in ("ID", "categorical", "numerical", "text", "timestamp"): + spec["stype"] = col["role"] + columns[col["name"]] = spec + inventory = source["tables"][name] + tables[name] = {"columns": columns, "primary_key": recipe["primary_key"], + "time_column": recipe["time_column"], "rows": inventory["rows"], + "files": sorted(item["path"] for item in inventory["files"])} + if not recipe["time_column"]: + tables[name]["availability"] = "timeless" + for item in inventory["files"]: + files[item["path"]] = {**item, "rows": sum(item["row_groups"])} + for rel in recipe["fkeys"]: + key = name + "_" + rel["column"] + relations[key] = {"child": name, "foreign_key": rel["column"], "parent": rel["table"]} + for name, task in source["tasks"].items(): + if task["type"] not in ("classification", "regression"): + raise ValueError("Unsupported generated task type") + target = {"dtype": "int64"} + if task["type"] == "classification": + target["classes"] = [0, 1] + tasks[name] = {"entity_table": task["entity_table"], "kind": task["type"], + "target": target, "label_horizon_us": 30 * DAY_US, + "label_available_column": "label_available_time"} + for split, item in task["artifacts"].items(): + tasks[name][split] = item["path"] + # The producer's task inventory omits size; report actual bytes. + files[item["path"]] = {**item, "bytes": safe_path(root, item["path"]).stat().st_size} + return {"format_version": 2, "logical_id": source["logical_id"], + "tables": tables, "relations": relations, "tasks": tasks, "files": files} + + +def workload_view(bundle, source): + """Resolve paths to directed layers and retain exact frozen instance order.""" + from .bundle import digest + source = deepcopy(source) + identifier = source.pop("workload_id", None) + if identifier != canonical_hash(source): + raise ValueError("Generated workload identity mismatch") + if source["logical_id"] != bundle.logical_id or source["snapshot_id"] != bundle.snapshot_id: + raise ValueError("Generated workload is bound to another snapshot") + if (source.get("history_interval") != "[anchor-lookback, anchor)" or + source.get("event_order") != ["timestamp DESC", "primary_key ASC"] or + source.get("identity_policy") != "deduplicate source IDs within each instance; preserve edges and per-instance occurrences"): + raise ValueError("Unsupported generated extraction policy") + timeless = {n for n, t in bundle.tables.items() if t.get("time_column") is None} + if set(source["timeless_dimensions"]) != timeless: + raise ValueError("Generated workload timeless dimensions differ from metadata") + layers, names = [], {bundle.manifest["tasks"][source["task"]]["entity_table"]} + for path in source["paths"]: + if len(path) < 2 or path[0] != bundle.manifest["tasks"][source["task"]]["entity_table"]: + raise ValueError("Generated paths must start at the task entity") + for depth, (src, dst) in enumerate(zip(path, path[1:])): + choices = [] + for name, rel in bundle.relations.items(): + if (rel["child"], rel["parent"]) == (src, dst): + choices.append((name, "forward")) + if (rel["parent"], rel["child"]) == (src, dst): + choices.append((name, "reverse")) + if len(choices) != 1: + raise ValueError(f"Missing or ambiguous directed path: {src} -> {dst}") + name, direction = choices[0] + step = {"relation": name, "direction": direction, + "fanout": source["forward_fk_fanout"] if direction == "forward" else source["event_fanout"]} + if len(layers) <= depth: + layers.append([]) + if step not in layers[depth]: + layers[depth].append(step) + names.update((src, dst)) + return {"version": 1, "logical_id": bundle.logical_id, "snapshot_id": bundle.snapshot_id, + "source_manifest_sha256": digest(bundle.root / "manifest.json"), "source_workload_id": identifier, + "task": source["task"], "prediction_cutoff": bundle.dataset_manifest["selection"]["prediction_cutoff"], + # Frozen cohorts/paths are reused, but the loader now deliberately + # compares local/native sampling. Retain the producer policy so the + # effective workload and report expose this semantic change. + "source_sampling_policy": {key: source[key] for key in + ("lookback_days", "history_interval", "event_order")}, + "selection": "local_native", "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", "layers": layers, + "projections": {n: source["projection"][n] for n in sorted(names)}, + "context_instance_ids": source["context_instance_ids"], + "prediction_instance_ids": source["prediction_instance_ids"], "encoding": "arrays"} diff --git a/packages/experimental/kumo-fm/src/kumo/fm/loader.py b/packages/experimental/kumo-fm/src/kumo/fm/loader.py new file mode 100644 index 000000000..44184c775 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/loader.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Dataset, index, request, and batch lifecycle.""" +from __future__ import annotations + +import json +from pathlib import Path + +from ._pytorch import KumoPytorchBackend +from .batch import PayloadBatch +from .bundle import Bundle +from .workload import Workload + + +def _select(available, selected, split): + if selected is None: + return list(available) + if not isinstance(selected, (list, tuple)): + raise TypeError(f"{split} must be an ordered sequence of instance IDs") + if any(type(value) is not int for value in selected): + raise ValueError(f"{split} instance IDs must be int64 values") + by_id = {seed["instance_id"]: seed for seed in available} + if len(set(selected)) != len(selected) or not set(selected) <= by_id.keys(): + raise ValueError(f"{split} has duplicate or unknown instance IDs") + return [by_id[value] for value in selected] + + +class PreparedRequest: + """Resolved anchors; every load call samples exactly once per included split.""" + + def __init__(self, loader, splits): + self._loader = loader + self._splits = splits + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + self._closed = True + + def load(self): + if self._closed or self._loader._closed: + raise RuntimeError("Prepared request or loader is closed") + return self._loader._load(self._splits) + + +class FMLoader: + """Open a validated bundle and its compatible reusable DiskGraph index.""" + + def __init__(self, dataset, spec, *, index, threads=2, verify_files=True): + bundle = Bundle(dataset.root if isinstance(dataset, Bundle) else Path(dataset), + verify_files=verify_files) + if isinstance(spec, Workload): + if spec.bundle.root != bundle.root: + raise ValueError("Workload is bound to another bundle") + workload = spec if spec.bundle is bundle else Workload(bundle, spec.spec) + else: + document = json.loads(Path(spec).read_text()) if isinstance(spec, (str, Path)) else spec + workload = Workload(bundle, document) + if verify_files: + bundle.validate(threads=threads) + self.workload = workload + self._backend = KumoPytorchBackend(workload, index, threads=threads) + self._closed = False + + @property + def preparation_mode(self): + return self._backend.preparation_mode + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + if not self._closed: + self._closed = True + self._backend.close() + + def materialize(self, *, context=None, predict=None): + if self._closed: + raise RuntimeError("Loader is closed") + available = self.workload.instances + splits = {} + for split, selected in (("context", context), ("predict", predict)): + seeds = _select(available[split], selected, split) + if seeds: + if len(seeds) > self.workload.budgets["max_instances"]: + raise ValueError("Instance budget exceeded") + self._backend.prepare_batch(seeds) + splits[split] = seeds + if not splits: + raise ValueError("Request needs at least one split") + return PreparedRequest(self, splits) + + def _load(self, splits): + ready = {split: (seeds, self._backend.extract(self.workload, seeds)) + for split, seeds in splits.items()} + return PayloadBatch(self.workload, ready) + + +def prepare(dataset=None, spec=None, *, index=None, threads=2, verify_files=True, + engine=None, sample=None, target=None, task=None, features=None, + classes=None, batch_size=1024, budgets=None, schema=None): + """Open a frozen bundle, or prepare FM over an adopted snapshot engine.""" + if engine is not None: + from ._snapshot import SnapshotFMLoader + if dataset is not None or spec is not None or index is not None: + raise ValueError("Snapshot and bundle inputs cannot be mixed") + return SnapshotFMLoader(engine=engine, sample=sample, target=target, + task=task, features=features, classes=classes, batch_size=batch_size, + budgets=budgets, schema=schema) + if schema is not None: + raise ValueError("schema is only valid with an adopted engine") + if dataset is None or spec is None or index is None: + raise ValueError("Bundle preparation needs dataset, spec, and index") + return FMLoader(dataset, spec, index=index, threads=threads, verify_files=verify_files) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_json.py b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py new file mode 100644 index 000000000..c29d81608 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared native arrays encoding; Python handles the small request envelope.""" +import json +from dataclasses import dataclass + +import pyarrow as pa +import pyarrow.compute as pc + +from ._phase import phase + + +@dataclass(frozen=True) +class EncodedRows: + """Internal JSON fragment produced only by the native Arrow encoder.""" + data: bytes + + +def encode_rows(table, max_bytes): + # Fail explicitly if a source checkout was not built; never silently time + # the reference encoder when native encoding was requested. + from ._native_json import encode_rows as native_encode_rows + + # Arrow's calendar formatter has implementation-specific behavior outside + # Python's year 1..9999 range. Preserve that existing behavior (including + # errors) with its native string kernel for unusual columns, while normal + # dates/timestamps use the direct C++ formatter. No per-cell Python values. + with phase('native_arrays_prepare', 'type_conversion'): + for index, field in enumerate(table.schema): + timestamp = pa.types.is_timestamp(field.type) + date = pa.types.is_date32(field.type) + if not (timestamp or date): + continue + column = table.column(index) + bounds = pc.min_max(column.cast(pa.int64() if timestamp else pa.int32())).as_py() + low, high = ((-62135596800000000, 253402300799999999) if timestamp else (-719162, 2932896)) + if bounds['min'] is not None and (bounds['min'] < low or bounds['max'] > high): + values = pc.strftime(column, format='%Y-%m-%dT%H:%M:%S' if timestamp else '%Y-%m-%d') + if timestamp: + values = pc.binary_join_element_wise(values, 'Z', '') + table = table.set_column(index, field.name, values) + return EncodedRows(native_encode_rows(table.__arrow_c_stream__(), max_bytes)) + + +def dumps(payload, max_bytes): + """Preserve json.dumps ordering/escaping while inserting encoded row arrays. + + Walk only the envelope dictionaries. Ordinary lists (including legacy row + lists) retain the reference JSON implementation. Native rows stay bytes. + """ + def parts(value): + if isinstance(value, EncodedRows): + yield value.data + elif isinstance(value, dict): + yield b'{' + for i, (key, item) in enumerate(value.items()): + if i: + yield b',' + yield json.dumps(key, ensure_ascii=True).encode('ascii') + yield b':' + yield from parts(item) + yield b'}' + else: + yield json.dumps(value, separators=(',', ':'), allow_nan=False).encode('ascii') + + fragments, size = [], 0 + for fragment in parts(payload): + size += len(fragment) + if size > max_bytes: + raise ValueError('Payload exceeds byte budget; reduce the workload batch') + fragments.append(fragment) + return b''.join(fragments) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py new file mode 100644 index 000000000..11960b5ee --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Inclusive whole-second graph cutoff for native anchor tables.""" +from datetime import timedelta + +from .workload import EPOCH + + +def inclusive_second(anchor): + return (anchor - EPOCH) // timedelta(seconds=1) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/workload.py b/packages/experimental/kumo-fm/src/kumo/fm/workload.py new file mode 100644 index 000000000..0bc617893 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/workload.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Frozen task instances and an explicit, bounded traversal policy.""" +from __future__ import annotations + +import json +import math +from datetime import date, datetime, time, timedelta, timezone +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from .bundle import TYPES, Bundle, identity + +UTC = timezone.utc +EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + + +def timestamp(value) -> datetime: + if isinstance(value, date) and not isinstance(value, datetime): + value = datetime.combine(value, time(), tzinfo=UTC) + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if not isinstance(value, datetime) or value.tzinfo is None: + raise ValueError("Timestamps must include a UTC offset") + return value.astimezone(UTC) + + +def validate_scalar(value, dtype: str, *, nullable: bool = False): + if value is None: + if nullable: + return + raise ValueError("Unexpected null value") + valid = ((dtype == "int64" and type(value) is int and -(2**63) <= value < 2**63) or + (dtype == "string" and isinstance(value, str)) or + (dtype == "bool" and type(value) is bool) or + (dtype == "float64" and type(value) in (int, float) and math.isfinite(value))) + if dtype == "timestamp[us]": + timestamp(value) + return + if not valid: + raise ValueError(f"Value does not match {dtype}: {value!r}") + + +class Workload: + def __init__(self, bundle: Bundle, spec: dict): + if getattr(bundle, "dataset_manifest", None) is not None and spec.get("version") != 1: + from .datasets import workload_view + spec = workload_view(bundle, spec) + self.bundle, self.spec = bundle, spec + if spec.get("version") != 1 or spec.get("logical_id") != bundle.logical_id or spec.get("snapshot_id") != bundle.snapshot_id: + raise ValueError("Workload is not bound to this logical dataset and physical snapshot") + self.task = bundle.manifest["tasks"][spec["task"]] + self.entity = self.task["entity_table"] + self.target = self.task["target"] + if self.entity not in bundle.tables or self.target["dtype"] not in ("string", "int64", "float64"): + raise ValueError("Unsupported entity/target type") + self.kind = self.task["kind"] + if self.kind not in ("binary_classification", "multiclass_classification", "classification", "regression"): + raise ValueError("Only classification and regression tasks are supported") + if self.kind == "regression": + if self.target["dtype"] not in ("float64", "int64"): + raise ValueError("Regression needs a numeric target") + else: + classes = self.target.get("classes", []) + if len(classes) < 2 or len(set(classes)) != len(classes): + raise ValueError("Classification needs distinct target classes") + if self.kind == "binary_classification" and len(classes) != 2: + raise ValueError("Binary classification needs exactly two classes") + for value in classes: + validate_scalar(value, self.target["dtype"]) + self.cutoff = timestamp(spec["prediction_cutoff"]) + if spec.get("lookback_us") is not None: + raise ValueError("lookback_us is unsupported; temporal_query uses lookback_seconds") + self.selection = spec.get("selection") + if self.selection not in ("local_native", "temporal_query", "native_multihop") or spec.get("deduplication") != "per_instance": + raise ValueError("Declare local_native, temporal_query or native_multihop selection and per-instance deduplication") + self.lookback_seconds = spec.get("lookback_seconds") + if self.lookback_seconds is not None: + if self.selection != "temporal_query": + raise ValueError("lookback_seconds requires temporal_query selection") + if type(self.lookback_seconds) is not int or not 0 < self.lookback_seconds < 2**63: + raise ValueError("lookback_seconds must be a positive int64 or null") + if spec.get("fanout_scope") != "per_source_relation_hop": + raise ValueError("fanout_scope must be per_source_relation_hop") + self.layers = spec["layers"] + frontier = {self.entity} + self.table_names, self.relation_names = {self.entity}, set() + if not isinstance(self.layers, list) or len(self.layers) > 32: + raise ValueError("Expected at most 32 explicit traversal layers") + for layer in self.layers: + next_frontier, seen = set(), set() + if not layer: + raise ValueError("Traversal layers cannot be empty") + for step in layer: + if step["direction"] not in ("forward", "reverse") or step["relation"] not in bundle.relations: + raise ValueError("Unknown directed relation") + name = (step["relation"], step["direction"]) + if name in seen: + raise ValueError("Repeated relation in one layer") + seen.add(name) + source, dest = bundle.endpoint(step) + if source not in frontier: + raise ValueError(f"Relation source is not reachable at this hop: {source}") + fanout = step["fanout"] + if fanout is not None and (type(fanout) is not int or fanout < 0): + raise ValueError("Fanout must be nonnegative or null (all)") + next_frontier.add(dest) + self.table_names.update((source, dest)) + self.relation_names.add(step["relation"]) + frontier = next_frontier + self.projections = spec["projections"] + if set(self.projections) != self.table_names: + raise ValueError("Projections must declare exactly the reachable tables, including empty tables") + for name, columns in self.projections.items(): + if len(columns) != len(set(columns)) or not set(columns) <= set(bundle.tables[name]["columns"]): + raise ValueError(f"Invalid projection for {name}") + if self.selection == "native_multihop": + from ._validation import validate_contract + validate_contract(self) + self.encoding = spec.get("encoding", "arrays") + if self.encoding not in ("arrays", "arrow_ipc"): + raise ValueError("Unknown output encoding") + self.budgets = {"max_instances": 10000, "max_rows_per_table": 1000000, + "max_payload_bytes": 30 * 1024**2, "max_decoded_arrow_bytes": 30 * 1024**2, + "max_columns": 500, "max_candidate_rows": 1000000, + **spec.get("budgets", {})} + if any(type(v) is not int or v < 1 for v in self.budgets.values()): + raise ValueError("Budgets must be positive integers") + self.instances = {split: self._instances(split) for split in ("context", "predict")} + self.id = identity(spec) + + @classmethod + def load(cls, bundle: Bundle, path: Path): + return cls(bundle, json.loads(Path(path).read_text())) + + def _instances(self, split: str) -> list[dict]: + file = self.task[split] + if file not in self.bundle.manifest["files"]: + raise ValueError("Task file is missing from the immutable inventory") + path = self.bundle.path(file) + parquet = pq.ParquetFile(path) + n = parquet.metadata.num_rows + if n != self.bundle.manifest["files"][file]["rows"]: + raise ValueError("Task row count differs from inventory") + selection = self.spec.get("context_instance_ids" if split == "context" else "prediction_instance_ids") + if selection is not None: + for value in selection: + validate_scalar(value, "int64") + if len(selection) != len(set(selection)): + raise ValueError("Duplicate selected instance IDs") + count = n if selection is None else len(selection) + if count < 1 or count > self.budgets["max_instances"]: + raise ValueError("Task instance count exceeds budget or is empty") + data = pq.read_table(path, filters=None if selection is None else [("instance_id", "in", selection)]) + available = self.task.get("label_available_column", "label_available_at") + if split == "context" and available != "label_available_at": + data = data.rename_columns(["label_available_at" if c == available else c for c in data.column_names]) + rows = data.to_pylist() + if selection is not None: + lookup = {row["instance_id"]: row for row in rows} + if len(rows) != len(lookup) or set(selection) != set(lookup): + raise ValueError("Selected task instances are missing or duplicated") + rows = [lookup[i] for i in selection] + expected = {"instance_id", "entity_key", "anchor_time"} + if split == "context": + expected |= {"target", "label_available_at"} + ids = set() + entity = self.bundle.tables[self.entity] + key_dtype = entity["columns"][entity["primary_key"]]["dtype"] + fields = {"instance_id": TYPES["int64"], "entity_key": TYPES[key_dtype], + "anchor_time": TYPES["timestamp[us]"]} + if split == "context": + fields.update(target=TYPES[self.target["dtype"]], label_available_at=TYPES["timestamp[us]"]) + schema = pa.schema([pa.field(c, t, nullable=False) for c, t in fields.items()]) + if not data.schema.equals(schema): + raise ValueError(f"Invalid {split} task schema; prediction targets must stay in labels.parquet") + horizon = self.task.get("label_horizon_us") + if horizon is not None and (type(horizon) is not int or horizon <= 0): + raise ValueError("Label horizon must be positive microseconds") + for row in rows: + if set(row) != expected: + raise ValueError(f"Unexpected {split} columns; prediction targets must stay in labels.parquet") + validate_scalar(row["instance_id"], "int64") + validate_scalar(row["entity_key"], key_dtype) + if row["instance_id"] in ids: + raise ValueError("Duplicate instance_id within a split") + ids.add(row["instance_id"]) + row["anchor_time"] = timestamp(row["anchor_time"]) + if split == "predict" and row["anchor_time"] != self.cutoff: + raise ValueError("Prediction anchors must share the workload cutoff") + if split == "context": + row["label_available_at"] = timestamp(row["label_available_at"]) + if not row["anchor_time"] <= row["label_available_at"] <= self.cutoff: + raise ValueError("Context label is unavailable at prediction cutoff") + if horizon is not None and row["label_available_at"] < row["anchor_time"] + timedelta(microseconds=horizon): + raise ValueError("Context label observation interval is incomplete") + validate_scalar(row["target"], self.target["dtype"]) + if self.kind != "regression" and row["target"] not in self.target["classes"]: + raise ValueError("Context target is outside the declared classes") + return rows + + def eligible(self, name: str, row: dict, seed: dict) -> bool: + tc = self.bundle.tables[name].get("time_column") + if tc is None: + return True + if row[tc] is None: + return False + value = timestamp(row[tc]) + anchor = seed["anchor_time"] + return value <= anchor + + def rank(self, name: str, row: dict): + table = self.bundle.tables[name] + tc = table.get("time_column") + micros = 0 if tc is None else (timestamp(row[tc]) - EPOCH) // timedelta(microseconds=1) + return -micros + + def neighbor_eligible(self, name: str, row: dict, seed: dict) -> bool: + """Window applies to destinations at every hop, never root availability.""" + if not self.eligible(name, row, seed): + return False + tc = self.bundle.tables[name].get("time_column") + if tc and self.lookback_seconds is not None: + age = (seed["anchor_time"] - timestamp(row[tc])) // timedelta(microseconds=1) + return age < self.lookback_seconds * 1000000 + return True + + def temporal_order(self, name: str, row: dict): + # query_temporal reverses both timestamp and Python-PK-repr ordering. + # Homogeneous int64/string PK types make its type-name tie key constant. + return (-self.rank(name, row), repr(row[self.bundle.tables[name]["primary_key"]])) diff --git a/packages/experimental/kumo-fm/tests/_fixture.py b/packages/experimental/kumo-fm/tests/_fixture.py new file mode 100644 index 000000000..e637a6ef5 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/_fixture.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tiny whole-second contract bundles, not a bulk generator. + +Timestamp values have one-second granularity. Parquet and FM serialization +retain timestamp[us] as the physical type, with zero subsecond components. +""" +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from kumo.fm.bundle import TYPES, Bundle, digest, identity + + +def scalar(value): + if isinstance(value, datetime): + return value.isoformat().replace('+00:00', 'Z') + return value + + +CUTOFF = datetime(2026, 2, 1, tzinfo=timezone.utc) +DAY_SECONDS = 86400 +DAY_US = DAY_SECONDS * 1000000 # Workload duration fields retain their schema units. +USER_A = 9007199254740993 +USER_B = -5 + + +def column(dtype, nullable=False, stype=None): + spec = {"dtype": dtype, "nullable": nullable} + if stype: + spec["stype"] = stype + return spec + + +def table(pk, columns, time_column=None): + return {"primary_key": pk, "columns": columns, "time_column": time_column, + **({"availability": "timeless"} if time_column is None else {})} + + +def commerce(): + tables = { + "users": table("user_id", {"user_id": column("int64"), "segment": column("string", stype="categorical"), "active": column("bool")}), + "interactions": table("interaction_id", {"interaction_id": column("int64"), "user_id": column("int64", True), + "item_id": column("string", True), "event_time": column("timestamp[us]", True), "amount": column("float64", True, "numerical")}, "event_time"), + "items": table("item_id", {"item_id": column("string"), "category_id": column("int64"), "description": column("string", stype="text")}), + "categories": table("category_id", {"category_id": column("int64"), "name": column("string", stype="categorical")}), + } + relations = {"interaction_user": {"child": "interactions", "foreign_key": "user_id", "parent": "users"}, + "interaction_item": {"child": "interactions", "foreign_key": "item_id", "parent": "items"}, + "item_category": {"child": "items", "foreign_key": "category_id", "parent": "categories"}} + events = [(11, USER_A, "shared", -1), (12, USER_A, "shared", -1), + (13, USER_A, "other", -1), (21, USER_B, "shared", -2), + (22, USER_B, "other", -10 * DAY_SECONDS), (23, USER_B, "other", -10 * DAY_SECONDS - 1), + (30, USER_A, "other", 0), (31, USER_A, "shared", DAY_SECONDS), + (40, USER_A, None, -3), (41, None, "shared", -1), + (42, USER_B, "other", None), (43, USER_A, "shared", -40 * DAY_SECONDS - 1), + (44, USER_A, "other", -20 * DAY_SECONDS)] + rows = { + "users": [{"user_id": USER_A, "segment": "a", "active": True}, + {"user_id": USER_B, "segment": "b", "active": False}, + {"user_id": 42, "segment": "quiet", "active": True}], + "interactions": [{"interaction_id": k, "user_id": u, "item_id": item, + "event_time": None if dt is None else CUTOFF + timedelta(seconds=dt), + "amount": None if k == 22 else 5.0} for k, u, item, dt in events], + "items": [{"item_id": "shared", "category_id": 7, "description": "shared item"}, + {"item_id": "other", "category_id": 7, "description": "other item"}, + {"item_id": "unused", "category_id": 8, "description": "unused"}], + "categories": [{"category_id": 7, "name": "common"}, {"category_id": 8, "name": "unused"}], + } + layers = [[{"relation": "interaction_user", "direction": "reverse", "fanout": 2}], + [{"relation": "interaction_item", "direction": "forward", "fanout": 1}], + [{"relation": "item_category", "direction": "forward", "fanout": 1}]] + return tables, relations, rows, "users", "interactions", "user_id", "event_time", layers + + +def marketo(): + facts = {"engagements": ("marketo_lead_id", "engagement_datetime"), + "metrics": ("marketo_lead_id", "trigger_date"), + "usecase_metrics": ("marketo_lead_id", "trigger_date"), + "conversions": ("marketo_lead_id", "u2_stage_date"), + "activities": ("id", "activity_date"), + "notes": ("marketo_lead_id", "source_creation_timestamp")} + tables = {"leads": table("Id", {"Id": column("int64"), "segment": column("string", stype="categorical")})} + rows = {"leads": [{"Id": USER_A, "segment": "a"}, {"Id": USER_B, "segment": "b"}, {"Id": 42, "segment": "quiet"}]} + relations, layer = {}, [] + for name, (fk, tc) in facts.items(): + tables[name] = table("event_id", {"event_id": column("int64"), fk: column("int64", True), + tc: column("timestamp[us]", True), "value": column("float64", True, "numerical")}, tc) + rows[name] = [] if name == "notes" else [ + {"event_id": k, fk: u, tc: CUTOFF + timedelta(seconds=dt), "value": None if k == 3 else 1.0} + for k, u, dt in [(1, USER_A, -1), (2, USER_A, -1), (3, USER_A, -1), + (4, USER_B, -10 * DAY_SECONDS), (5, USER_B, 0), (6, None, -1), + (7, USER_A, -20 * DAY_SECONDS)]] + relations[name + "_lead"] = {"child": name, "foreign_key": fk, "parent": "leads"} + layer.append({"relation": name + "_lead", "direction": "reverse", "fanout": 2}) + return tables, relations, rows, "leads", "engagements", "marketo_lead_id", "engagement_datetime", [layer] + + +def write_bundle(root, tables, relations, rows, tasks, *, recipe, reverse_layout=False, row_group_size=3): + """Small-fixture writer; imported/bulk bundles use the documented same format.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=False) + inventory = {} + logical_tables = {} + for name, spec in tables.items(): + path = root / "tables" / name / "part-00000.parquet" + path.parent.mkdir(parents=True) + schema = pa.schema([pa.field(c, TYPES[s["dtype"]], s["nullable"]) for c, s in spec["columns"].items()]) + data = list(reversed(rows[name])) if reverse_layout else rows[name] + pq.write_table(pa.Table.from_pylist(data, schema=schema), path, row_group_size=row_group_size) + relative = str(path.relative_to(root)) + spec.update(rows=len(data), files=[relative]) + ordered = sorted(rows[name], key=lambda r: r[spec["primary_key"]]) + logical_tables[name] = json.loads(json.dumps(ordered, default=scalar)) + meta = pq.ParquetFile(path).metadata + inventory[relative] = {"sha256": digest(path), "rows": len(data), "bytes": path.stat().st_size, + "row_groups": [meta.row_group(i).num_rows for i in range(meta.num_row_groups)]} + task_specs = {} + for name, task in tasks.items(): + task_specs[name] = {k: v for k, v in task.items() if k not in ("context", "predict", "labels")} + entity = tables[task["entity_table"]] + key_type = TYPES[entity["columns"][entity["primary_key"]]["dtype"]] + for split in ("context", "predict", "labels"): + path = root / "tasks" / name / f"{split}.parquet" + path.parent.mkdir(parents=True, exist_ok=True) + if split == "labels": + schema = pa.schema([pa.field("instance_id", pa.int64(), False), pa.field("target", TYPES[task["target"]["dtype"]], False)]) + else: + fields = [pa.field("instance_id", pa.int64(), False), pa.field("entity_key", key_type, False), + pa.field("anchor_time", TYPES["timestamp[us]"], False)] + if split == "context": + fields += [pa.field("target", TYPES[task["target"]["dtype"]], False), pa.field("label_available_at", TYPES["timestamp[us]"], False)] + schema = pa.schema(fields) + pq.write_table(pa.Table.from_pylist(task[split], schema=schema), path) + relative = str(path.relative_to(root)) + inventory[relative] = {"sha256": digest(path), "rows": len(task[split]), "bytes": path.stat().st_size} + task_specs[name][split] = relative + logical = {"recipe": recipe, "tables": logical_tables, + "schema": {n: {k: v for k, v in t.items() if k not in ("files", "rows")} for n, t in tables.items()}, + "relations": relations} + manifest = {"format_version": 2, "logical_id": identity(logical), "recipe": recipe, + "tables": tables, "relations": relations, "tasks": task_specs, "files": inventory, + "writer": {"pyarrow": pa.__version__, "row_group_size": row_group_size, "reverse_layout": reverse_layout}} + (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + # Perform real validation before publishing the completion marker. + provisional = Bundle.__new__(Bundle) + provisional.root, provisional.manifest = root.resolve(), manifest + provisional.tables, provisional.relations = tables, relations + report = {"status": "passed", **provisional.validate()} + (root / "validation.json").write_text(json.dumps(report, indent=2) + "\n") + (root / "_COMPLETE.json").write_text(json.dumps({"manifest_sha256": digest(root / "manifest.json"), + "validation_sha256": digest(root / "validation.json")}, indent=2) + "\n") + return Bundle(root) + + +def generate(root: Path, preset="commerce_v1", *, reverse_layout=False, row_group_size=3): + if preset not in ("commerce_v1", "marketo_v1"): + raise ValueError("Choose commerce_v1 or marketo_v1") + tables, relations, rows, entity, fact, fk, tc, layers = commerce() if preset == "commerce_v1" else marketo() + tasks = {} + context = [{"instance_id": 0, "entity_key": USER_A, "anchor_time": CUTOFF - timedelta(days=40)}, + {"instance_id": 1, "entity_key": USER_B, "anchor_time": CUTOFF - timedelta(days=40)}] + predict = [{"instance_id": i, "entity_key": key, "anchor_time": CUTOFF} + for i, key in [(100, USER_A), (101, USER_B), (102, USER_A), (103, 42)]] + + def count(seed): + return sum(1 for r in rows[fact] if r[fk] == seed["entity_key"] and r[tc] is not None + and seed["anchor_time"] <= r[tc] < seed["anchor_time"] + timedelta(days=30)) + for kind, target in [("binary_classification", {"dtype": "string", "classes": ["no", "yes"]}), + ("regression", {"dtype": "int64"})]: + def label(seed): + return count(seed) if kind == "regression" else ("yes" if count(seed) else "no") + tasks[kind] = {"entity_table": entity, "kind": kind, "target": target, + "label_horizon_us": 30 * DAY_US, + "context": [{**s, "target": label(s), "label_available_at": s["anchor_time"] + timedelta(days=30)} for s in context], + "predict": predict, "labels": [{"instance_id": s["instance_id"], "target": label(s)} for s in predict]} + bundle = write_bundle(root, tables, relations, rows, tasks, + recipe={"generator": "tiny_contract_v3", "topology": preset, + "seed": 42, "timestamp_granularity": "seconds"}, + reverse_layout=reverse_layout, row_group_size=row_group_size) + workload_dir = Path(root) / "workloads" + workload_dir.mkdir() + hops = (2, 3) if preset == "commerce_v1" else (1,) + for kind in tasks: + for depth in hops: + selected = layers[:depth] + names = {entity} + for layer in selected: + for step in layer: + names.update(bundle.endpoint(step)) + spec = {"version": 1, "logical_id": bundle.logical_id, "snapshot_id": bundle.snapshot_id, + "task": kind, "prediction_cutoff": scalar(CUTOFF), + "selection": "local_native", "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", "layers": selected, + "projections": {n: list(tables[n]["columns"]) for n in sorted(names)}, "encoding": "arrays"} + (workload_dir / f"{kind}-{depth}hop.json").write_text(json.dumps(spec, indent=2) + "\n") + return bundle diff --git a/packages/experimental/kumo-fm/tests/test_batch.py b/packages/experimental/kumo-fm/tests/test_batch.py new file mode 100644 index 000000000..cee65012f --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_batch.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +import pyarrow as pa +import pytest + +from kumo.fm.batch import PayloadBatch + +BIG_ID = 9007199254740993 + + +def workload(): + tables = { + "users": {"primary_key": "user_id", "columns": { + "user_id": {"dtype": "int64", "nullable": False}, + "segment": {"dtype": "string", "nullable": True}}}, + "events": {"primary_key": "event_id", "columns": { + "event_id": {"dtype": "int64", "nullable": False}, + "user_id": {"dtype": "int64", "nullable": True}}}, + } + bundle = SimpleNamespace(tables=tables, relations={ + "event_user": {"child": "events", "foreign_key": "user_id", "parent": "users"}}) + return SimpleNamespace( + bundle=bundle, entity="users", table_names=set(tables), relation_names={"event_user"}, + projections={"users": ["segment"], "events": ["event_id", "user_id"]}, + target={"dtype": "int64", "classes": [0, 1]}, kind="classification", + budgets={"max_columns": 10, "max_rows_per_table": 10, + "max_decoded_arrow_bytes": 100000, "max_payload_bytes": 100000}) + + +def request(work): + anchor = datetime(2026, 2, 1, tzinfo=timezone.utc) + context = [{"instance_id": 7, "entity_key": BIG_ID, "anchor_time": anchor, "target": 1}] + predict = [{"instance_id": 8, "entity_key": BIG_ID, "anchor_time": anchor}] + + def related(instance_id): + return { + "users": pa.table({"instance_id": pa.array([instance_id], pa.int64()), + "user_id": pa.array([BIG_ID], pa.int64()), + "segment": pa.array(["a"])}), + "events": pa.table({"instance_id": pa.array([instance_id], pa.int64()), + "event_id": pa.array([17], pa.int64()), + "user_id": pa.array([BIG_ID], pa.int64()), + "__fm_edge_event_user": pa.array([BIG_ID], pa.int64())}), + } + return PayloadBatch(work, {"context": (context, related(7)), + "predict": (predict, related(8))}) + + +def test_retained_tables_encode_both_formats_without_mutation(): + batch = request(workload()) + before = batch.tables("context")["related_tables"]["events"] + metadata = batch.metadata + metadata["task"]["kind"] = "changed" + assert batch.metadata["task"]["kind"] == "classification" + direct = batch.as_arrow_request() + arrays = json.loads(batch.encode("arrays")) + ipc = json.loads(batch.encode("arrow_ipc")) + assert arrays["context"]["related_tables"]["events"]["rows"] == [[7, 17, str(BIG_ID), str(BIG_ID)]] + for split in ("context", "predict"): + for name in ("users", "events"): + encoded = ipc[split]["related_tables"][name] + decoded = pa.ipc.open_stream(base64.b64decode(encoded["payload_base64"])).read_all() + assert decoded.equals(direct[split]["related_tables"][name]) + assert batch.tables("context")["related_tables"]["events"].equals(before) + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + assert batch.logical_arrow_bytes == sum( + frame.nbytes for split in batch.splits for frame in ( + batch.tables(split)["instance_table"], + *batch.tables(split)["related_tables"].values())) + with pytest.raises(ValueError, match="Unknown encoding"): + batch.encode("invalid") + assert batch.encode("arrays") + + +def test_logical_and_wire_budgets_are_independent(): + work = workload() + work.budgets["max_decoded_arrow_bytes"] = 1 + with pytest.raises(ValueError, match="Decoded Arrow budget"): + request(work) + work = workload() + work.budgets["max_payload_bytes"] = 1 + batch = request(work) + assert batch.tables("context")["instance_table"].num_rows == 1 + with pytest.raises(ValueError, match="byte budget"): + batch.encode("arrays") + with pytest.raises(ValueError, match="byte budget"): + batch.encode("arrow_ipc") diff --git a/packages/experimental/kumo-fm/tests/test_connector_snapshot.py b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py new file mode 100644 index 000000000..dbc15460c --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Installed-wheel connector → snapshot → FM, including exported reuse.""" +import base64 +import hashlib +import json +import shutil +from datetime import datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from kumo import fm +from kumo.pytorch import api + +kumo = pytest.importorskip("kumo.engine") +from kumoapi.typing import Stype # noqa: E402 + +from kumo.engine._scratch import get_scratch_dir # noqa: E402 + + +def inventory(root): + return {str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in root.rglob("*") if path.is_file()} + + +def at(second): + return datetime.fromtimestamp(second, tz=timezone.utc) + + +def test_connector_export_reuse_and_aliased_relation(tmp_path): + source = tmp_path / "source" + source.mkdir() + pq.write_table(pa.table({"id": [-5, 9007199254740993]}), source / "users.parquet") + pq.write_table(pa.table({"id": [1, 2, 3, 4], + "user": [-5, -5, -5, 9007199254740993], + "at": pa.array([10, 20, 30, 10], pa.timestamp("s"))}), + source / "events.parquet") + connector = kumo.FileConnector(root_dir=str(source)) + graph = kumo.Graph(tables={ + "users": kumo.Table.from_source_table(connector["users.parquet"], primary_key="id"), + "events": kumo.Table.from_source_table(connector["events.parquet"], + primary_key="id", time_column="at").with_column("user", stype=Stype.ID), + }, edges=[kumo.Edge(src_table="events", fkey="user", dst_table="users")]) + previous = get_scratch_dir() + try: + kumo.configure(scratch_dir=tmp_path / "producer") + snapshot = graph.snapshot() + original_hashes = inventory(snapshot.index_path) + + def run(snapshot, work): + engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=work, + num_threads=2) + physical = engine.schema.relations[0] + alias = api.RelationSpec("events_selected", physical.left_table, + physical.left_key, physical.right_table, physical.right_key) + schema = api.DataSchema(engine.schema.tables, (alias,)) + future = api.Aggregate(api.Rows( + path=(alias.name,), time=api.TimeWindow("at", start=0, stop=10)), "count") + layer = api.GraphLayer((api.GraphSample( + alias.name, direction="reverse", fanout=2, strategy="recent"),)) + sample = api.SampleSpec( + entity=api.EntitySpec("users"), targets={"next": future}, + neighborhoods={"history": api.NeighborhoodSpec((layer,))}) + with fm.prepare(engine=engine, schema=schema, sample=sample, target="next", + task="regression", features={"users": [], "events": ["at"]}, + batch_size=1) as loader: + batch = loader.load(context=fm.Context(pairs=[(-5, at(15), 88), + (-5, at(15), 89), (9007199254740993, at(15), 90)]), + query=fm.Query(pairs=[(-5, at(40), 91), + (9007199254740993, at(40), 92)])) + context = batch.tables("context") + assert [(row["instance_id"], row["__fm_target"]) + for row in context["instance_table"].to_pylist()] == [ + (88, 1.0), (89, 1.0), (90, 0.0)] + rows = context["related_tables"]["events"].to_pylist() + assert [(row["instance_id"], row["id"], row["__fm_edge_events_selected"]) + for row in rows] == [(88, 1, -5), (89, 1, -5), + (90, 4, 9007199254740993)] + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + arrays, ipc = batch.encode("arrays"), batch.encode("arrow_ipc") + assert arrays == batch.encode("arrays") and ipc == batch.encode("arrow_ipc") + wire_arrays, wire_ipc = json.loads(arrays), json.loads(ipc) + assert wire_arrays["context"]["related_tables"]["events"]["rows"][2][0] == 90 + assert str(9007199254740993) in arrays.decode() + for split in batch.splits: + direct = batch.tables(split) + entries = {"instance_table": direct["instance_table"], + **direct["related_tables"]} + for name, table in entries.items(): + encoded = (wire_ipc[split]["instance_table"] if name == "instance_table" + else wire_ipc[split]["related_tables"][name]) + decoded = pa.ipc.open_stream(base64.b64decode( + encoded["payload_base64"])).read_all() + assert decoded.equals(table) + return batch.tables("predict")["related_tables"]["events"] + first = run(snapshot, tmp_path / "work1") + assert inventory(snapshot.index_path) == original_hashes + exports = kumo.FileConnector(root_dir=str(tmp_path / "exports")) + location = snapshot.export(kumo.SnapshotOutputConfig(exports, "saved")) + shutil.rmtree(source) + shutil.rmtree(tmp_path / "producer") + kumo.configure(scratch_dir=tmp_path / "consumer") + reopened = kumo.GraphSnapshot.load_from_connector(exports, path=location) + exported_hashes = inventory(reopened.index_path) + second = run(reopened, tmp_path / "work2") + assert first.equals(second) + assert inventory(reopened.index_path) == exported_hashes + finally: + kumo.configure(scratch_dir=previous) diff --git a/packages/experimental/kumo-fm/tests/test_contract.py b/packages/experimental/kumo-fm/tests/test_contract.py new file mode 100644 index 000000000..3203de9a8 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_contract.py @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace + +import pytest + +from kumo.fm._pytorch import compatibility_issues, contract_for +from kumo.fm.loader import _select + + +def test_unsupported_sampling_is_explicit(): + bundle = SimpleNamespace( + tables={"events": {"time_column": None}}, + endpoint=lambda step: ("users", "events")) + workload = SimpleNamespace( + bundle=bundle, selection="native_multihop", + layers=[[{"relation": "event_user", "direction": "reverse", "fanout": 3}]]) + assert "uniform sampling" in compatibility_issues(workload)[0] + with pytest.raises(ValueError, match="Cannot preserve FM sampling"): + contract_for(workload) + workload.layers = [[{"relation": "event_user", "direction": "reverse", "fanout": 0}]] + assert "empty graph layer" in compatibility_issues(workload)[0] + workload.selection = "temporal_query" + assert "native_multihop" in compatibility_issues(workload)[0] + + +def test_anchor_selection_keeps_order_and_rejects_unknown_ids(): + available = [{"instance_id": 1}, {"instance_id": 2}] + assert _select(available, [2, 1], "predict") == [available[1], available[0]] + with pytest.raises(ValueError, match="duplicate or unknown"): + _select(available, [1, 1], "predict") + with pytest.raises(ValueError, match="duplicate or unknown"): + _select(available, [3], "predict") diff --git a/packages/experimental/kumo-fm/tests/test_integration.py b/packages/experimental/kumo-fm/tests/test_integration.py new file mode 100644 index 000000000..870c1c017 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_integration.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from copy import deepcopy +from datetime import timedelta + +import pyarrow.parquet as pq +from _fixture import CUTOFF, USER_A, USER_B, generate, write_bundle + +from kumo import fm + + +def fixture(tmp_path): + original = generate(tmp_path / "original") + tables = deepcopy(original.tables) + rows = {name: pq.read_table(original.paths(name)).to_pylist() for name in tables} + rows["interactions"] = [ + {"interaction_id": event_id, "user_id": user, "item_id": "shared", + "event_time": CUTOFF + timedelta(days=days), "amount": 5.0} + for event_id, user, days in ((1, USER_A, -41), (2, USER_A, -1), + (3, USER_B, -41), (4, USER_B, -1)) + ] + tasks = deepcopy(original.manifest["tasks"]) + for task in tasks.values(): + for split in ("context", "predict", "labels"): + task[split] = pq.read_table(original.path(task[split])).to_pylist() + bundle = write_bundle(tmp_path / "supported", tables, deepcopy(original.relations), + rows, tasks, recipe={"generator": "kumo_fm_integration"}) + spec = json.loads((original.root / "workloads/binary_classification-2hop.json").read_text()) + spec.update(logical_id=bundle.logical_id, snapshot_id=bundle.snapshot_id, + selection="native_multihop", layers=spec["layers"][:1], + projections={name: list(bundle.tables[name]["columns"]) + for name in ("users", "interactions")}) + spec["layers"][0][0]["fanout"] = 1 + return bundle, spec + + +def test_installed_wheel_samples_once_and_batch_survives_close(tmp_path): + bundle, spec = fixture(tmp_path) + index = tmp_path / "index" + with fm.prepare(bundle, spec, index=index) as loader: + assert loader.preparation_mode == "index_build" + with loader.materialize() as request: + batch = request.load() + loader._backend.extract = lambda *_: (_ for _ in ()).throw( + AssertionError("encoding resampled")) + arrays = batch.encode("arrays") + ipc = batch.encode("arrow_ipc") + assert arrays and ipc + assert batch.tables("predict")["instance_table"].num_rows == 4 + context_events = batch.tables("context")["related_tables"]["interactions"] + query_events = batch.tables("predict")["related_tables"]["interactions"] + assert set(context_events["interaction_id"].to_pylist()) == {1, 3} + assert set(query_events["interaction_id"].to_pylist()) == {2, 4} + assert set(zip(query_events["instance_id"].to_pylist(), + query_events["interaction_id"].to_pylist())) == { + (100, 2), (101, 4), (102, 2)} + assert all(value is not None for value in query_events["__fm_edge_interaction_user"].to_pylist()) + with fm.prepare(bundle, spec, index=index) as loader: + assert loader.preparation_mode == "cached_index_open" + with loader.materialize(context=[], predict=[100]) as request: + single = request.load() + assert single.splits == ("predict",) + assert single.tables("predict")["instance_table"].num_rows == 1 diff --git a/packages/experimental/kumo-fm/tests/test_sdm_adapter.py b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py new file mode 100644 index 000000000..b15920bfc --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys +from pathlib import Path +from types import SimpleNamespace + +import pyarrow as pa +from test_batch import request, workload + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "examples/local_sdm")) +from adapter import _compatible_buffers, arrow_to_sdm # noqa: E402 + + +def test_sdm_bridge_uses_public_arrow_conversion(monkeypatch): + calls = [] + + class TableTensor: + def __init__(self, table, stypes): + self.table, self.stypes = table, stypes + + @classmethod + def from_arrow(cls, table, *, stypes, device): + assert device == "cpu" + calls.append(table) + return cls(table, stypes) + + def drop_columns(self, name): + return TableTensor(self.table.drop([name]), { + key: value for key, value in self.stypes.items() if key != name}) + + def __getitem__(self, names): + return TableTensor(self.table.select(names), { + key: self.stypes[key] for key in names}) + + class RelatedTables: + def __init__(self, *, tables, relationships, task_links): + self.tables, self.relationships, self.task_links = tables, relationships, task_links + + monkeypatch.setitem(sys.modules, "sdm", SimpleNamespace( + TableTensor=TableTensor, RelatedTables=RelatedTables)) + batch = request(workload()) + source = batch.tables("context")["related_tables"]["events"] + result = arrow_to_sdm(batch) + assert len(calls) == 6 + assert source.equals(batch.tables("context")["related_tables"]["events"]) + assert result["y_context"].stypes == {"__fm_target": "categorical"} + assert "__fm_target" not in result["x_query"].stypes + assert result["related_context_tables"].relationships[0]["left_columns"] == [ + "instance_id", "__fm_edge_event_user"] + assert result["related_query_tables"].task_links[0]["task_columns"] == [ + "instance_id", "__fm_entity_key"] + + +def test_buffer_compatibility_only_repairs_incompatible_columns(): + writable = pa.table({"name": pa.array(["a", None], type=pa.string())}) + assert _compatible_buffers(writable) is writable + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, writable.schema) as writer: + writer.write_table(writable) + read_only = pa.ipc.open_stream(sink.getvalue()).read_all() + assert not read_only["name"].chunk(0).buffers()[2].is_mutable + repaired = _compatible_buffers(read_only) + assert repaired.equals(read_only) + assert repaired["name"].chunk(0).buffers()[2].is_mutable diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_api.py b/packages/experimental/kumo-fm/tests/test_snapshot_api.py new file mode 100644 index 000000000..1e3f817e1 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_snapshot_api.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Snapshot input validation before any native graph is sampled.""" +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +from kumo.fm import Context, Query +from kumo.fm._snapshot import ( + SnapshotFMLoader, + _rows, + _schema_view, + _selected_graph, + _target_horizon, +) +from kumo.pytorch import api + + +def schema(): + return api.DataSchema( + tables=( + api.TableSpec("users", "user_id", {"user_id": api.DType("int64")}, + time_column=api.NON_TEMPORAL), + api.TableSpec("orders", "order_id", { + "order_id": api.DType("int64"), "user_id": api.DType("int64"), + "at": api.DType("timestamp", unit="s", timezone="UTC"), + "amount": api.DType("float64")}, time_column="at"), + ), + relations=(api.RelationSpec("alias", "users", "user_id", "orders", "user_id"),), + ) + + +def test_population_order_and_repeated_occurrences(): + context = _rows(Context(entities=[7, 8], times=["2025-01-01T00:00:00Z", + "2025-02-01T00:00:00Z"]), + "context", "int64") + assert [(row["instance_id"], row["entity_key"]) for row in context] == [ + (0, 7), (1, 8), (2, 7), (3, 8)] + query = _rows(Query(pairs=[(7, "2025-03-01T00:00:00Z", 9007199254740993), + (7, "2025-03-01T00:00:00Z", -1)]), "predict", "int64") + assert [row["instance_id"] for row in query] == [9007199254740993, -1] + assert query[0]["anchor_time"] == datetime(2025, 3, 1, tzinfo=timezone.utc) + with pytest.raises(ValueError, match="Duplicate occurrence"): + _rows(Query(pairs=[(7, "2025-03-01T00:00:00Z", 1), + (8, "2025-03-01T00:00:00Z", 1)]), "predict", "int64") + + +def test_schema_alias_projection_and_target_window(): + bundle = _schema_view(schema()) + assert bundle.relations["alias"] == {"child": "orders", "parent": "users", + "foreign_key": "user_id"} + target = api.Aggregate(api.Rows(path=("alias",), + time=api.TimeWindow("at", start=0, stop=30)), "count") + assert _target_horizon(target) == 30 + layer = api.GraphLayer((api.GraphSample( + "alias", direction="reverse", fanout=2, strategy="recent"),)) + sample = api.SampleSpec(entity=api.EntitySpec("users"), targets={"next": target}, + neighborhoods={"history": api.NeighborhoodSpec((layer,))}) + name, fields, tables, relations, projections = _selected_graph( + schema(), sample, {"users": [], "orders": ["amount"]}, bundle) + assert (name, tables, relations) == ("history", {"users", "orders"}, {"alias"}) + assert projections == {"orders": ["order_id", "amount"], "users": ["user_id"]} + assert {item.field for item in fields["orders"].fields} == { + "order_id", "amount", "at", "user_id"} + with pytest.raises(ValueError, match="uniformly into timeless"): + reverse_layer = api.GraphLayer((api.GraphSample( + "alias", direction="forward", fanout=2, strategy="recent"),)) + reverse_sample = api.SampleSpec( + entity=api.EntitySpec("orders"), + neighborhoods={"history": api.NeighborhoodSpec((reverse_layer,))}) + _selected_graph(schema(), reverse_sample, {"orders": [], "users": []}, bundle) + + +def test_context_labels_join_by_occurrence_id_across_batches(): + class Task: + def __init__(self, values): + self.values = values + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def batches(self, **_): + for identifier, value in self.values: + yield SimpleNamespace(metadata={"sample_ids": [identifier]}, + targets={"label": {"values": SimpleNamespace(tolist=lambda: [value]), + "valid": SimpleNamespace(tolist=lambda: [True])}}) + + loader = object.__new__(SnapshotFMLoader) + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3), (11, 9)])) + loader._batch_size = 1 + loader._classes = None + assert loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) == { + 22: 3.0, 11: 9.0} + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3)])) + with pytest.raises(ValueError, match="Missing FM context labels"): + loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3), (22, 9)])) + with pytest.raises(ValueError, match="duplicated"): + loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_integration.py b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py new file mode 100644 index 000000000..c9fdd5a86 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py @@ -0,0 +1,141 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Integrated multi-layer FM sampling over a reusable native index.""" +from datetime import datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from kumo import fm +from kumo.pytorch import api + +UTC = timezone.utc +LARGE = 9007199254740993 + + +def when(day): + return datetime(2025, 1, day, tzinfo=UTC) + + +def build_index(tmp_path): + rows = { + "users": [{"user_id": LARGE}, {"user_id": -5}], + "orders": [ + {"order_id": 11, "user_id": LARGE, "at": when(1)}, + {"order_id": 12, "user_id": LARGE, "at": when(2)}, + {"order_id": 13, "user_id": -5, "at": when(2)}, + {"order_id": 14, "user_id": LARGE, "at": when(15)}, + ], + "payments": [ + {"payment_id": 21, "order_id": 11, "at": when(3)}, + {"payment_id": 22, "order_id": 12, "at": when(3)}, + {"payment_id": 23, "order_id": 13, "at": when(3)}, + ], + "audits": [ + {"audit_id": 31, "payment_id": 21, "at": when(4)}, + {"audit_id": 32, "payment_id": 22, "at": when(4)}, + {"audit_id": 33, "payment_id": 23, "at": when(4)}, + ], + "notes": [ + {"note_id": 41, "audit_id": 31, "at": when(5)}, + {"note_id": 42, "audit_id": 32, "at": when(5)}, + {"note_id": 43, "audit_id": 33, "at": when(5)}, + ], + } + keys = {"users": "user_id", "orders": "order_id", "payments": "payment_id", + "audits": "audit_id", "notes": "note_id"} + sources, tables = {}, [] + for name, values in rows.items(): + path = tmp_path / f"{name}.parquet" + table = pa.Table.from_pylist(values) + pq.write_table(table, path) + sources[name] = api.Parquet(str(path)) + tables.append(api.TableSpec(name, keys[name], + {field.name: api.DType.from_arrow(field.type) for field in table.schema}, + time_column="at" if name != "users" else api.NON_TEMPORAL)) + relations = ( + api.RelationSpec("order_user", "orders", "user_id", "users", "user_id"), + api.RelationSpec("payment_order", "payments", "order_id", "orders", "order_id"), + api.RelationSpec("audit_payment", "audits", "payment_id", "payments", "payment_id"), + api.RelationSpec("note_audit", "notes", "audit_id", "audits", "audit_id"), + ) + schema = api.DataSchema(tuple(tables), relations) + first_layer = api.GraphLayer((api.GraphSample( + "order_user", direction="reverse", fanout=2, strategy="recent"),)) + graph = api.SampleSpec(entity=api.EntitySpec("users"), + neighborhoods={"history": api.NeighborhoodSpec((first_layer,))}) + contract = api.DataLoaderSpec(schema, graph, api.BatchSpec(inputs={ + "graph": api.Input(api.NeighborhoodSource("history"), api.Graph(node_fields={ + name: api.Fields(tuple(api.Encode(column, api.Raw()) for column in keys_and_types)) + for name, keys_and_types in ((table.name, table.columns) for table in tables + if table.name in ("users", "orders")) + }), None)}, batching=api.Batching(max_samples=2))) + index = tmp_path / "index" + with api.prepare(contract=contract, dataset=api.Dataset(sources), + engine=api.DiskGraphEngine(index=index, num_threads=2)): + pass + return index, schema + + +@pytest.mark.parametrize("depth", [2, 3, 4]) +def test_snapshot_multihop_and_occurrence_alignment(tmp_path, depth): + index, schema = build_index(tmp_path) + work = tmp_path / "consumer" + engine = api.DiskGraphEngine.from_index(index, work_dir=work, num_threads=2) + assert engine.schema == schema + names = ("order_user", "payment_order", "audit_payment", "note_audit") + future = api.Aggregate(api.Rows( + path=(names[0],), time=api.TimeWindow("at", start=0, stop=7*86400)), "count") + layers = tuple(api.GraphLayer((api.GraphSample( + name, direction="reverse", fanout=2, strategy="recent"),)) + for name in names[:depth]) + sample = api.SampleSpec(entity=api.EntitySpec("users"), targets={"future": future}, + neighborhoods={"history": api.NeighborhoodSpec(layers)}) + tables = ("users", "orders", "payments", "audits", "notes")[:depth+1] + features = {name: [] for name in tables} + context = fm.Context(pairs=[(LARGE, when(10), 100), + (LARGE, when(10), 101), (-5, when(10), 102)]) + query = fm.Query(pairs=[(LARGE, when(25), 200), (-5, when(25), 201)]) + with fm.prepare(engine=engine, sample=sample, target="future", task="regression", + features=features, batch_size=2) as loader: + batch = loader.load(context=context, query=query) + instance = batch.tables("context")["instance_table"].to_pylist() + assert [(row["instance_id"], row["__fm_target"]) for row in instance] == [ + (100, 1.0), (101, 1.0), (102, 0.0)] + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + related = batch.tables("context")["related_tables"] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 100] == [11, 12] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 101] == [11, 12] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 102] == [13] + expected_nodes = { + "payments": ("payment_id", [(100, 21), (100, 22), (101, 21), + (101, 22), (102, 23)]), + "audits": ("audit_id", [(100, 31), (100, 32), (101, 31), + (101, 32), (102, 33)]), + "notes": ("note_id", [(100, 41), (100, 42), (101, 41), + (101, 42), (102, 43)]), + } + for name, (key, expected) in expected_nodes.items(): + if name in related: + assert [(row["instance_id"], row[key]) for row in + related[name].to_pylist()] == expected + expected_edges = { + "order_user": [LARGE, LARGE, LARGE, LARGE, -5], + "payment_order": [11, 12, 11, 12, 13], + "audit_payment": [21, 22, 21, 22, 23], + "note_audit": [31, 32, 31, 32, 33], + } + for step in names[:depth]: + child = {"order_user": "orders", "payment_order": "payments", + "audit_payment": "audits", "note_audit": "notes"}[step] + assert "__fm_edge_" + step in related[child].column_names + assert related[child]["__fm_edge_" + step].null_count == 0 + assert related[child]["__fm_edge_" + step].to_pylist() == expected_edges[step] + assert batch.encode("arrays") == batch.encode("arrays") + assert batch.encode("arrow_ipc") == batch.encode("arrow_ipc") diff --git a/pyproject.toml b/pyproject.toml index 965fa9195..1000a73dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ members = [ "packages/kumo-engine", "packages/kumo-sdk", "packages/experimental/kumo-pytorch", + "packages/experimental/kumo-fm", "tools/glove-6b-50d", ] diff --git a/scripts/check_python_license_metadata.py b/scripts/check_python_license_metadata.py index 8cbca3031..5ece86ddf 100644 --- a/scripts/check_python_license_metadata.py +++ b/scripts/check_python_license_metadata.py @@ -90,6 +90,8 @@ class DerivedSourceRequirement: ("LICENSE", )), DistributionSpec("packages/experimental/kumo-pytorch/pyproject.toml", "kumo-pytorch", ("LICENSE", )), + DistributionSpec("packages/experimental/kumo-fm/pyproject.toml", + "kumo-fm", ("LICENSE", )), DistributionSpec("tools/glove-6b-50d/pyproject.toml", "kumo-glove-6b-50d", ("LICENSE", "THIRD-PARTY.txt"), notice_artifacts=("glove-6b-50d-bundled-data", )), diff --git a/scripts/gen_third_party.py b/scripts/gen_third_party.py index 154f11f65..3ce4d0c82 100755 --- a/scripts/gen_third_party.py +++ b/scripts/gen_third_party.py @@ -56,6 +56,7 @@ "kumo-snowflake", "kumo-engine", "kumo-pytorch", + "kumo-fm", "kumo-glove-6b-50d", } NATIVE_DISTRIBUTIONS = {"diskgraph"} diff --git a/test/scripts/test_python_license_metadata.py b/test/scripts/test_python_license_metadata.py index f2fce3a86..78735936a 100644 --- a/test/scripts/test_python_license_metadata.py +++ b/test/scripts/test_python_license_metadata.py @@ -52,7 +52,7 @@ def synthetic_spec( def test_repository_manifest_inventory_passes() -> None: - assert len(check_python_license_metadata.DISTRIBUTIONS) == 15 + assert len(check_python_license_metadata.DISTRIBUTIONS) == 16 assert check_python_license_metadata.validate_repository( REPOSITORY_ROOT) == [] diff --git a/tests/public_setup/Dockerfile.dockerignore b/tests/public_setup/Dockerfile.dockerignore index 598f31e29..67897beea 100644 --- a/tests/public_setup/Dockerfile.dockerignore +++ b/tests/public_setup/Dockerfile.dockerignore @@ -16,6 +16,7 @@ !bin/kumo-pred !packages/diskgraph/pyproject.toml !packages/experimental/kumo-pytorch/pyproject.toml +!packages/experimental/kumo-fm/pyproject.toml !tools/glove-6b-50d/pyproject.toml # Public fake accounts and policies for the local storage fixture. !docker/compose/config/minio-test/** diff --git a/uv.lock b/uv.lock index f46d2a930..af21516ca 100644 --- a/uv.lock +++ b/uv.lock @@ -57,6 +57,7 @@ members = [ "kumo-core", "kumo-databricks", "kumo-engine", + "kumo-fm", "kumo-gcs", "kumo-glove-6b-50d", "kumo-pql", @@ -1601,6 +1602,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] +[[package]] +name = "duckdb" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +requires-python = ">=3.11.0" +sdist = { url = "https://files.pythonhosted.org/packages/c2/7c/dfdf5e9cebefd032f57f09b224993e861ecb55f61e36c709544b04b44f70/duckdb-2.0.0.tar.gz", hash = "sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f", size = 16771077, upload-time = "2026-10-21T12:51:13.908572Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/57/ec7c1a01ab43fc13c55c17cc54a5740886e04ae830aa420ac00614b1420a/duckdb-2.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a3b38d8cdbef10007c64b65efc4087f799c4dc606785aa5f2a425ed13658ca01", size = 37006608, upload-time = "2026-10-21T12:50:38.611878Z" }, + { url = "https://files.pythonhosted.org/packages/84/36/e1ea89acd74917ca654d5adc278765f0116e07540abf4134d5cf9f527363/duckdb-2.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:9bcc9921be0e1b2404b96a7fb447a8b80fe2f3a919cdc976ca6c39951cc8f2f5", size = 19691094, upload-time = "2026-10-21T12:50:41.839172Z" }, + { url = "https://files.pythonhosted.org/packages/4b/81/2895b9ee3858dae7a57f791d7dbd4f116e9fa8fd3c71b9a377dbd4d292a7/duckdb-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61a244faae9fd787e30f4c76888769ec49e79736c434f698e8178a00af8348f0", size = 17603089, upload-time = "2026-10-21T12:50:44.167966Z" }, + { url = "https://files.pythonhosted.org/packages/04/95/0c8b5b95ac9e14ee83090a78048fbb23fb9f8abacd63e11a5eee7fcb31e1/duckdb-2.0.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcdf10dfbc2c1a6bdc5b2a94bba5b15481e552c5745c2af1994617b0eb6bc750", size = 21637946, upload-time = "2026-10-21T12:50:46.444674Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a1/9c3780faada6e6a3edc5f9364cf19810e6e1ea90a7d5fd8fffc48b42c269/duckdb-2.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a3601121c7cc50d0b70412995634682fafcb58a0156d5d663dfd0b3bbe30a33", size = 24351483, upload-time = "2026-10-21T12:50:48.902652Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/0729ac5d9857e5f0bde7262961f80e578a29945a444169886094cc9c69f4/duckdb-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a387d4907b535a4bd82ba8aede6809d0bf023dfbb21362832f3c9848aea7743c", size = 15370691, upload-time = "2026-10-21T12:50:51.590941Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/392991ec9dcb22811c7029009184a910bd5cb69fbb7a626f36f151215535/duckdb-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:62bea9a1786be98d2906197c67a4089cc8b34d66d19e87e5543d8201e81323ad", size = 15731407, upload-time = "2026-10-21T12:50:53.889460Z" }, + { url = "https://files.pythonhosted.org/packages/41/28/1fb52f9ea10205a55a07c32e37cd4fd68fa9d1dc2520a12405ec9bb59cac/duckdb-2.0.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9", size = 37003294, upload-time = "2026-10-21T12:50:56.489773Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/fbe92644709fb9162cc51753f92f892b1f333658079d1f11efaed43341d2/duckdb-2.0.0-cp312-abi3-macosx_10_15_x86_64.whl", hash = "sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd", size = 19689207, upload-time = "2026-10-21T12:50:59.331542Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/4d23938223626980cdcf18fd6b0dbf040823b5cf15beabbf4ab7ee276631/duckdb-2.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef", size = 17597612, upload-time = "2026-10-21T12:51:01.818477Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/985fb9d93d7f9f8e82092ddfe05ec8c9a1d0df7bdb0f562a10ced30e3e8f/duckdb-2.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa", size = 21655920, upload-time = "2026-10-21T12:51:04.360238Z" }, + { url = "https://files.pythonhosted.org/packages/cc/33/01ad2535ee897db2bad70cbc800655167c53afd0b955eed84f0759c2f417/duckdb-2.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431", size = 24371005, upload-time = "2026-10-21T12:51:06.860701Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/9e8ca1242b98f04041bef09686ecfe098b41f820aa23240f63f360b64eaf/duckdb-2.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a", size = 15368860, upload-time = "2026-10-21T12:51:09.395345Z" }, + { url = "https://files.pythonhosted.org/packages/37/81/6ea5dc1050f86ec4dd038a2774766b42247ba6104e7785f13a0c600e2b33/duckdb-2.0.0-cp312-abi3-win_arm64.whl", hash = "sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873", size = 15726489, upload-time = "2026-10-21T12:51:11.643301Z" }, +] + [[package]] name = "ec2-metadata" version = "2.10.0" @@ -3705,6 +3729,36 @@ requires-dist = [ ] provides-extras = ["test", "docs"] +[[package]] +name = "kumo-fm" +version = "0.1.0a1" +source = { editable = "packages/experimental/kumo-fm" } +dependencies = [ + { name = "duckdb" }, + { name = "kumo-pytorch" }, + { name = "numpy" }, + { name = "pyarrow" }, +] + +[package.optional-dependencies] +relbench = [ + { name = "pyyaml" }, +] +test = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "duckdb", specifier = "==2.0.0" }, + { name = "kumo-pytorch", editable = "packages/experimental/kumo-pytorch", specifier = ">=0.1.0a1" }, + { name = "numpy", specifier = ">=1.26,<3" }, + { name = "pyarrow", specifier = "==24.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8,<10" }, + { name = "pyyaml", marker = "extra == 'relbench'", specifier = ">=6,<7" }, +] +provides-extras = ["relbench", "test"] + [[package]] name = "kumo-pytorch" source = { editable = "packages/experimental/kumo-pytorch" }