From 3b92023ebe1646c1911a5347651184c11144b82e Mon Sep 17 00:00:00 2001 From: Alex Porter <102990994+am-porter@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:36:29 -0800 Subject: [PATCH 01/12] Preserve extension identity in path resolution errors --- packages/diskgraph/diskgraph-index/src/graph.rs | 4 ++-- .../python/tests/test_runtime_batch_iterator_e2e.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/diskgraph/diskgraph-index/src/graph.rs b/packages/diskgraph/diskgraph-index/src/graph.rs index 982d7d782..85d75878d 100644 --- a/packages/diskgraph/diskgraph-index/src/graph.rs +++ b/packages/diskgraph/diskgraph-index/src/graph.rs @@ -1241,8 +1241,8 @@ impl GraphIndexReader { // All opening paths share this validation, including direct native // callers and legacy relative references that traverse symlinks. let parent = fs::canonicalize(parent_root).context("resolve parent index path")?; - let subdir = - fs::canonicalize(parent_root.join(subdir_rel)).context("resolve extension index path")?; + let subdir = fs::canonicalize(parent_root.join(subdir_rel)) + .with_context(|| format!("resolve extension `{namespace}` index path: {subdir_rel}"))?; ensure!( subdir != parent, "extension path must not identify the base index" diff --git a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py index a65da050c..47bbb8aaa 100644 --- a/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py +++ b/packages/diskgraph/diskgraph-pybind/python/tests/test_runtime_batch_iterator_e2e.py @@ -414,6 +414,8 @@ def test_runtime_rejects_non_extension_paths(seed_table_fixture, tmp_path, kind, stage = parent / "train_tables" / ("base-alias-" + entrypoint) stage.symlink_to(parent, target_is_directory=True) match = "base index" if kind in ("base", "base_alias", "legacy_alias") else "extension" + if kind == "missing": + match = "extension `stage`" with pytest.raises(diskgraph.DiskgraphSystemError, match=match): if entrypoint == "native": from diskgraph._c import RuntimeIndex as NativeRuntimeIndex From 1fb5bbaee816108c559748347c9f2b9d67bb61f7 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Wed, 18 Nov 2026 01:02:27 +0000 Subject: [PATCH 02/12] Add experimental FM payload loader over DiskGraph --- .github/workflows/kumo-fm.yml | 65 ++ ARCHITECTURE.md | 1 + benchmark/fm-payloads/README.md | 44 ++ benchmark/fm-payloads/make_fixture.py | 25 + .../reports/omnistation-smoke.json | 139 ++++ benchmark/fm-payloads/runner.py | 151 +++++ docs/README.md | 1 + packages/experimental/kumo-fm/LICENSE | 204 ++++++ packages/experimental/kumo-fm/MANIFEST.in | 2 + packages/experimental/kumo-fm/README.md | 85 +++ .../kumo-fm/examples/local_sdm/README.md | 28 + .../kumo-fm/examples/local_sdm/adapter.py | 68 ++ .../examples/local_sdm/requirements.lock | 3 + .../examples/local_sdm/walkthrough.ipynb | 123 ++++ packages/experimental/kumo-fm/pyproject.toml | 38 ++ packages/experimental/kumo-fm/setup.py | 8 + .../kumo-fm/src/kumo/fm/__init__.py | 5 + .../kumo-fm/src/kumo/fm/_assembly.py | 145 +++++ .../kumo-fm/src/kumo/fm/_contract.py | 57 ++ .../src/kumo/fm/_dataset_support/__init__.py | 3 + .../src/kumo/fm/_dataset_support/config.py | 280 ++++++++ .../src/kumo/fm/_dataset_support/generate.py | 335 ++++++++++ .../fm/_dataset_support/presets/marketo.json | 504 +++++++++++++++ .../src/kumo/fm/_dataset_support/relbench.py | 603 ++++++++++++++++++ .../src/kumo/fm/_dataset_support/storage.py | 259 ++++++++ .../src/kumo/fm/_dataset_support/tasks.py | 241 +++++++ .../src/kumo/fm/_dataset_support/validate.py | 282 ++++++++ .../kumo-fm/src/kumo/fm/_encoding.py | 66 ++ .../kumo-fm/src/kumo/fm/_native_json.cpp | 238 +++++++ .../kumo-fm/src/kumo/fm/_phase.py | 6 + .../kumo-fm/src/kumo/fm/_pytorch.py | 270 ++++++++ .../kumo-fm/src/kumo/fm/_validation.py | 42 ++ .../experimental/kumo-fm/src/kumo/fm/batch.py | 110 ++++ .../kumo-fm/src/kumo/fm/bundle.py | 163 +++++ .../kumo-fm/src/kumo/fm/datasets.py | 99 +++ .../kumo-fm/src/kumo/fm/loader.py | 106 +++ .../kumo-fm/src/kumo/fm/native_json.py | 68 ++ .../kumo-fm/src/kumo/fm/native_selection.py | 8 + .../kumo-fm/src/kumo/fm/workload.py | 231 +++++++ .../experimental/kumo-fm/tests/_fixture.py | 198 ++++++ .../experimental/kumo-fm/tests/test_batch.py | 90 +++ .../kumo-fm/tests/test_contract.py | 31 + .../kumo-fm/tests/test_integration.py | 63 ++ .../kumo-fm/tests/test_sdm_adapter.py | 63 ++ pyproject.toml | 1 + uv.lock | 54 ++ 46 files changed, 5606 insertions(+) create mode 100644 .github/workflows/kumo-fm.yml create mode 100644 benchmark/fm-payloads/README.md create mode 100644 benchmark/fm-payloads/make_fixture.py create mode 100644 benchmark/fm-payloads/reports/omnistation-smoke.json create mode 100644 benchmark/fm-payloads/runner.py create mode 100644 packages/experimental/kumo-fm/LICENSE create mode 100644 packages/experimental/kumo-fm/MANIFEST.in create mode 100644 packages/experimental/kumo-fm/README.md create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/README.md create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/adapter.py create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/requirements.lock create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb create mode 100644 packages/experimental/kumo-fm/pyproject.toml create mode 100644 packages/experimental/kumo-fm/setup.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/__init__.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_assembly.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_contract.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_encoding.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_phase.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_validation.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/batch.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/bundle.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/datasets.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/loader.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/native_json.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/native_selection.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/workload.py create mode 100644 packages/experimental/kumo-fm/tests/_fixture.py create mode 100644 packages/experimental/kumo-fm/tests/test_batch.py create mode 100644 packages/experimental/kumo-fm/tests/test_contract.py create mode 100644 packages/experimental/kumo-fm/tests/test_integration.py create mode 100644 packages/experimental/kumo-fm/tests/test_sdm_adapter.py diff --git a/.github/workflows/kumo-fm.yml b/.github/workflows/kumo-fm.yml new file mode 100644 index 000000000..652de67bb --- /dev/null +++ b/.github/workflows/kumo-fm.yml @@ -0,0 +1,65 @@ +name: kumo-fm + +on: + pull_request: + paths: + - 'packages/experimental/kumo-fm/**' + - 'packages/experimental/kumo-pytorch/**' + - 'packages/diskgraph/**' + - 'uv.lock' + - '.github/workflows/kumo-fm.yml' + push: + branches: [main] + paths: + - 'packages/experimental/kumo-fm/**' + - 'packages/experimental/kumo-pytorch/**' + - 'packages/diskgraph/**' + - 'uv.lock' + - '.github/workflows/kumo-fm.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + diskgraph-wheel: + uses: ./.github/workflows/_diskgraph-wheel.yml + + package-and-wheel: + needs: diskgraph-wheel + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - uses: ./.github/actions/kumo-pytorch-setup + with: + diskgraph-wheel-artifact: diskgraph-wheel + - name: Test the source package against the current native stack + run: | + uv sync --locked --inexact --no-install-package diskgraph --no-install-package torch \ + --package kumo-fm --extra test + uv run --no-sync --package kumo-fm python -m pytest -q \ + packages/experimental/kumo-fm/tests + - name: Build both finished wheels + run: | + uv build --wheel --package kumo-pytorch --out-dir "$RUNNER_TEMP/dist" + uv build --sdist --package kumo-fm --out-dir "$RUNNER_TEMP/dist" + uv build --wheel "$RUNNER_TEMP"/dist/kumo_fm-*.tar.gz --out-dir "$RUNNER_TEMP/dist" + - name: Install finished wheels outside the checkout + working-directory: ${{ runner.temp }} + run: | + uv venv --python 3.12 fresh + uv pip install --python fresh/bin/python --torch-backend cpu \ + /tmp/diskgraph-wheel/diskgraph-*.whl dist/kumo_pytorch-*.whl \ + dist/kumo_fm-*.whl 'pytest>=8,<10' + uv pip check --python fresh/bin/python + - name: Run tests using only installed runtime packages + run: | + mkdir "$RUNNER_TEMP/stage" + cp -R packages/experimental/kumo-fm/tests \ + packages/experimental/kumo-fm/examples "$RUNNER_TEMP/stage/" + cd "$RUNNER_TEMP/stage" + ../fresh/bin/python -I -c 'import kumo.fm, kumo.pytorch, diskgraph; print(kumo.fm.__file__)' + ../fresh/bin/python -m pytest -q tests diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ba3fab5af..eddcf7445 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -12,6 +12,7 @@ This monorepo contains Kumo's graph machine-learning application, Python package | Service Python SDK | `packages/kumo-sdk/` | Python client for a configured Kumo service (`kumoai_studio`) | | Local Python SDK | `packages/kumo-engine/` | In-process SDK surface (`kumo.engine`) for local execution with local or remote data sources | | Shared runtime | `packages/kumo-core/` | Service-independent data processing, training, evaluation, and prediction used by the service and local SDK | +| Experimental model data | `packages/experimental/kumo-pytorch/`, `packages/experimental/kumo-fm/` | Public columnar sampling and FM-specific Arrow payload assembly | | Connectors | `kumo/connector/` | Data-warehouse and file-source integrations | | Data pipeline | `kumo/data/` | Export, cache, and DiskGraph-backed table generation | | Query language | `packages/kumo-pql/`, `kumo/pquery/` | Parser and semantic validation package; service query and table-generation orchestration | diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md new file mode 100644 index 000000000..6db60cf8b --- /dev/null +++ b/benchmark/fm-payloads/README.md @@ -0,0 +1,44 @@ +# FM payload comparison + +`runner.py` compares the installed `kumo-fm` wheel with the pinned +fm-dataloader DuckDB reference on the same frozen bundle, workload, and +selected instances. Install the reference checkout only in the benchmark +environment; it is not a `kumo-fm` runtime dependency. + +Run correctness first, then warmups and interleaved measured draws: + +```bash +python make_fixture.py /tmp/fm-case +python runner.py --bundle /tmp/fm-case/supported \ + --workload /tmp/fm-case/workload.json --index /tmp/fm-case/index \ + --draws 5 --output /tmp/fm-report.json +``` + +For an existing bundle and workload: + +```bash +python runner.py --bundle /data/bundle --workload /data/workload.json \ + --index /data/fm-index --draws 7 --output /tmp/fm-report.json +``` + +The workload must be eligible for the public `kumo.pytorch` recent-sampling +path. Uniform/timeless and empty-layer workloads are reported as excluded, +not silently rewritten. Reports include source revisions, dependency +versions, machine metadata, row counts, logical Arrow bytes, body bytes, and +raw per-phase timings. Retain the bundle, workload, wheel hashes, index +provenance, and report together when publishing a comparison. + +## Omnistation smoke comparison + +[Raw report](reports/omnistation-smoke.json) records five interleaved draws +after two warmups on a four-event, one-hop synthetic graph. Both encodings +decoded identically to the DuckDB reference and produced the same body byte +counts. Median phase times in milliseconds: + +| Encoding | DuckDB reference full payload | DiskGraph load and assembly | Encoding retained batch | +| --- | ---: | ---: | ---: | +| Arrays | 40.691 | 3.643 | 0.731 | +| Arrow IPC | 40.765 | 3.457 | 0.446 | + +The backend paths and measured phase boundaries differ. This small smoke +case validates parity and timing instrumentation, not general throughput. diff --git a/benchmark/fm-payloads/make_fixture.py b/benchmark/fm-payloads/make_fixture.py new file mode 100644 index 000000000..08de153e7 --- /dev/null +++ b/benchmark/fm-payloads/make_fixture.py @@ -0,0 +1,25 @@ +"""Create the frozen smoke case used by the benchmark and SDM notebook.""" + +import json +import sys +from pathlib import Path + +TESTS = Path(__file__).resolve().parents[2] / "packages/experimental/kumo-fm/tests" +sys.path.insert(0, str(TESTS)) + +from test_integration import fixture # noqa: E402 + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: python make_fixture.py /path/to/new/case") + root = Path(sys.argv[1]).resolve() + root.mkdir(parents=True, exist_ok=False) + bundle, workload = fixture(root) + (root / "workload.json").write_text(json.dumps(workload, indent=2) + "\n") + print(f"bundle: {bundle.root}") + print(f"workload: {root / 'workload.json'}") + + +if __name__ == "__main__": + main() diff --git a/benchmark/fm-payloads/reports/omnistation-smoke.json b/benchmark/fm-payloads/reports/omnistation-smoke.json new file mode 100644 index 000000000..2d6f173f5 --- /dev/null +++ b/benchmark/fm-payloads/reports/omnistation-smoke.json @@ -0,0 +1,139 @@ +{ + "bundle_manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", + "workload_sha256": "df440df955a2edcae0f64acc5b5061272afd855006f944afbc0108a0b0eaf3c0", + "wheel_sha256": { + "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "c55014ca6868058f9ddbd4efdcc57032eba24e9152dc445e6e4d95e56edc5818" + }, + "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", + "engine_revision": "9637e9cc9e2da877959b3c6b9db88c0fc7e35f3f", + "reference_revision": "b4516394e98929c5b4752745866d5e3138276255", + "checkout_dirty": true, + "machine": { + "platform": "Linux-7.0.0-1010-aws-x86_64-with-glibc2.39", + "processor": "x86_64", + "python": "3.11.15" + }, + "versions": { + "kumo-fm": "0.1.0a1", + "kumo-pytorch": "0.1.0a1", + "diskgraph": "0.2.0", + "fm-dataloader": "0.1.0", + "duckdb": "2.0.0", + "pyarrow": "24.0.0", + "numpy": "2.4.6" + }, + "cases": [ + { + "encoding": "arrays", + "context_rows": 2, + "predict_rows": 4, + "logical_arrow_bytes": 584, + "new_body_bytes": 3374, + "reference_body_bytes": 3374, + "draws": [ + { + "reference_full_ms": 41.190471, + "new_load_ms": 3.740637, + "new_encode_ms": 0.731258 + }, + { + "reference_full_ms": 40.420899, + "new_load_ms": 3.376303, + "new_encode_ms": 0.704164 + }, + { + "reference_full_ms": 40.677103, + "new_load_ms": 3.644739, + "new_encode_ms": 0.743447 + }, + { + "reference_full_ms": 41.676955, + "new_load_ms": 3.449625, + "new_encode_ms": 0.73143 + }, + { + "reference_full_ms": 40.691307, + "new_load_ms": 3.642878, + "new_encode_ms": 0.74569 + } + ] + }, + { + "encoding": "arrow_ipc", + "context_rows": 2, + "predict_rows": 4, + "logical_arrow_bytes": 584, + "new_body_bytes": 8859, + "reference_body_bytes": 8859, + "draws": [ + { + "reference_full_ms": 41.208853, + "new_load_ms": 3.451474, + "new_encode_ms": 0.460802 + }, + { + "reference_full_ms": 39.633872, + "new_load_ms": 3.457388, + "new_encode_ms": 0.443918 + }, + { + "reference_full_ms": 40.712534, + "new_load_ms": 3.589053, + "new_encode_ms": 0.422048 + }, + { + "reference_full_ms": 42.041882, + "new_load_ms": 3.405239, + "new_encode_ms": 0.445812 + }, + { + "reference_full_ms": 40.765063, + "new_load_ms": 3.608553, + "new_encode_ms": 0.46231 + } + ] + } + ], + "warmups": 2, + "draws_per_encoding": 5, + "graph": { + "layers": [ + [ + { + "relation": "interaction_user", + "direction": "reverse", + "fanout": 1 + } + ] + ], + "table_rows": { + "interactions": 4, + "users": 3 + }, + "relations": [ + "interaction_user" + ] + }, + "eligibility": "supported", + "exclusion_reasons": [], + "index_marker_sha256": "58ff9e961cf4bf9ed58598f4ae8ea483c3a0af5e0a819efdc5679b90eec236d5", + "selected_rows": { + "context": { + "interactions": 2, + "users": 2 + }, + "predict": { + "interactions": 3, + "users": 4 + } + }, + "selected_edges": { + "context": { + "interaction_user": 2 + }, + "predict": { + "interaction_user": 3 + } + }, + "correctness": "passed" +} diff --git a/benchmark/fm-payloads/runner.py b/benchmark/fm-payloads/runner.py new file mode 100644 index 000000000..d1c8965e9 --- /dev/null +++ b/benchmark/fm-payloads/runner.py @@ -0,0 +1,151 @@ +"""Correctness-gated, interleaved FM payload timing against the DuckDB reference.""" +import argparse +import hashlib +import importlib.metadata +import json +import platform +import random +import subprocess +import time +from urllib.parse import urlparse +from pathlib import Path + +from kumo import fm +from kumo.fm.bundle import Bundle +from kumo.fm.workload import Workload +from kumo.fm._pytorch import compatibility_issues + + +def revision(path): + result = subprocess.run(["git", "rev-parse", "HEAD"], cwd=path, capture_output=True, text=True) + return result.stdout.strip() if result.returncode == 0 else None + + +def digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def installed_revision(name): + distribution = importlib.metadata.distribution(name) + source = json.loads(distribution.read_text("direct_url.json") or "{}").get("url") + return revision(Path(urlparse(source).path)) if source and source.startswith("file://") else None + + +def measure(callback): + start = time.perf_counter_ns() + result = callback() + return result, (time.perf_counter_ns() - start) / 1e6 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--workload", type=Path, required=True) + parser.add_argument("--index", type=Path, required=True) + parser.add_argument("--draws", type=int, default=7) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--wheel", type=Path, action="append", default=[]) + args = parser.parse_args() + if args.draws < 1 or args.warmups < 1: + parser.error("draws and warmups must be positive") + + from fm_dataloader.bundle import Bundle as OldBundle + from fm_dataloader.multihop_sql import DuckDBMultihopBackend + from fm_dataloader.payload import build_payload, decode_payload + from fm_dataloader.workload import Workload as OldWorkload + + old_bundle = OldBundle(args.bundle) + old_workload = OldWorkload.load(old_bundle, args.workload) + new_bundle = Bundle(args.bundle) + new_workload = Workload.load(new_bundle, args.workload) + issues = compatibility_issues(new_workload) + report = { + "bundle_manifest_sha256": digest(args.bundle / "manifest.json"), + "workload_sha256": digest(args.workload), + "wheel_sha256": {path.name: digest(path) for path in args.wheel}, + "snapshot_id": new_bundle.snapshot_id, + "engine_revision": revision(Path(__file__).resolve().parents[2]), + "reference_revision": installed_revision("fm-dataloader"), + "checkout_dirty": bool(subprocess.run( + ["git", "status", "--porcelain"], cwd=Path(__file__).resolve().parents[2], + capture_output=True, text=True).stdout.strip()), + "machine": {"platform": platform.platform(), "processor": platform.processor(), + "python": platform.python_version()}, + "versions": {name: importlib.metadata.version(name) for name in + ("kumo-fm", "kumo-pytorch", "diskgraph", "fm-dataloader", + "duckdb", "pyarrow", "numpy")}, + "cases": [], + "warmups": args.warmups, + "draws_per_encoding": args.draws, + "graph": {"layers": new_workload.layers, + "table_rows": {name: new_bundle.tables[name]["rows"] + for name in new_workload.table_names}, + "relations": sorted(new_workload.relation_names)}, + "eligibility": "excluded" if issues else "supported", + "exclusion_reasons": issues, + } + if issues: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + "\n") + return + old_backend = DuckDBMultihopBackend(old_bundle, native=True) + try: + with fm.prepare(new_bundle, new_workload, index=args.index) as loader: + report["index_marker_sha256"] = digest(args.index / "fm_snapshot.json") + with loader.materialize() as request: + batch = request.load() + report["selected_rows"] = { + split: {name: table.num_rows for name, table in + batch.tables(split)["related_tables"].items()} + for split in batch.splits} + report["selected_edges"] = { + split: { + relation: len(batch.tables(split)["related_tables"][rel["child"]][ + "__fm_edge_" + relation]) + - batch.tables(split)["related_tables"][rel["child"]][ + "__fm_edge_" + relation].null_count + for relation, rel in new_bundle.relations.items() + if relation in new_workload.relation_names} + for split in batch.splits} + for encoding in ("arrays", "arrow_ipc"): + old_body = build_payload(old_backend, old_workload, encoding=encoding) + new_body = batch.encode(encoding) + if decode_payload(old_body) != decode_payload(new_body): + raise AssertionError(f"{encoding} differs from DuckDB reference") + report["cases"].append({ + "encoding": encoding, + "context_rows": batch.tables("context")["instance_table"].num_rows, + "predict_rows": batch.tables("predict")["instance_table"].num_rows, + "logical_arrow_bytes": batch.logical_arrow_bytes, + "new_body_bytes": len(new_body), + "reference_body_bytes": len(old_body), + "draws": [], + }) + for iteration in range(args.warmups + args.draws): + order = report["cases"][:] + random.Random(iteration).shuffle(order) + for case in order: + encoding = case["encoding"] + old_body, old_ms = measure( + lambda: build_payload(old_backend, old_workload, encoding=encoding)) + new_batch, load_ms = measure(request.load) + new_body, encode_ms = measure(lambda: new_batch.encode(encoding)) + if decode_payload(old_body) != decode_payload(new_body): + raise AssertionError(f"{encoding} changed during timing") + if iteration >= args.warmups: + case["draws"].append({ + "reference_full_ms": old_ms, + "new_load_ms": load_ms, + "new_encode_ms": encode_ms, + }) + finally: + old_backend.close() + args.output.parent.mkdir(parents=True, exist_ok=True) + report["correctness"] = "passed" + args.output.write_text(json.dumps(report, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/docs/README.md b/docs/README.md index 80ba64bc9..6dfacafc0 100644 --- a/docs/README.md +++ b/docs/README.md @@ -34,6 +34,7 @@ guides for their interfaces and development workflows. | API schemas | [kumo-api](../packages/kumo-api/README.md) | | Service Python client | [kumo-sdk](../packages/kumo-sdk/README.md) | | In-process Python interface | [kumo-engine](../packages/kumo-engine/README.md) | +| Experimental model batches and FM payloads | [kumo-pytorch](../packages/experimental/kumo-pytorch/README.md) and [kumo-fm](../packages/experimental/kumo-fm/README.md) | | Shared execution runtime | [kumo-core](../packages/kumo-core/README.md) | | Query language | [kumo-pql](../packages/kumo-pql/README.md) and [existing instructions](../packages/kumo-pql/AGENTS.md) | | UI component library | [catalog](../packages/kumo-ui-catalog/README.md) and [existing component guide](../packages/kumo-ui-catalog/CLAUDE.md) | diff --git a/packages/experimental/kumo-fm/LICENSE b/packages/experimental/kumo-fm/LICENSE new file mode 100644 index 000000000..d1593aaa1 --- /dev/null +++ b/packages/experimental/kumo-fm/LICENSE @@ -0,0 +1,204 @@ +Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/experimental/kumo-fm/MANIFEST.in b/packages/experimental/kumo-fm/MANIFEST.in new file mode 100644 index 000000000..f44b7b194 --- /dev/null +++ b/packages/experimental/kumo-fm/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include examples/local_sdm *.py *.ipynb *.md *.lock +recursive-include tests *.py diff --git a/packages/experimental/kumo-fm/README.md b/packages/experimental/kumo-fm/README.md new file mode 100644 index 000000000..73e9c0e1c --- /dev/null +++ b/packages/experimental/kumo-fm/README.md @@ -0,0 +1,85 @@ +# Kumo FM payload loader + +Experimental `kumo-fm` builds the existing Kumo Relational HTTP request from +public `kumo.pytorch` columnar graph batches backed by DiskGraph. Sampling and +FM assembly happen once per `PreparedRequest.load()`. The returned +`PayloadBatch` retains typed Arrow tables; calls to `encode("arrays")` and +`encode("arrow_ipc")` serialize those same tables without sampling. + +Build and install `kumo-pytorch` and DiskGraph from the same engine revision, +then build and install this package's wheel. The example under +`examples/local_sdm` additionally needs SDM and a compatible pretrained +checkpoint; SDM is not a runtime dependency. + +```bash +uv build --wheel --package kumo-pytorch --out-dir dist +uv build --sdist --package kumo-fm --out-dir dist +uv build --wheel dist/kumo_fm-*.tar.gz --out-dir dist +uv pip install --python /path/to/venv/bin/python --torch-backend cpu \ + /path/to/matching/diskgraph-*.whl dist/kumo_pytorch-*.whl dist/kumo_fm-*.whl +``` + +```python +from kumo import fm + +with fm.prepare("/data/bundle", "/data/workload.json", index="/data/fm-index") as loader: + with loader.materialize(context=[101, 102], predict=[201]) as request: + batch = request.load() + context = batch.tables("context") + print(context["instance_table"].schema) + arrays_body = batch.encode("arrays") + ipc_body = batch.encode("arrow_ipc") +``` + +The optional `context` and `predict` lists contain ordered instance IDs +already validated by the workload. Omit a list to use that split's frozen +instances. Pass `[]` to omit a split. A direct SDM prediction needs both. +`materialize()` prepares the requested anchors; `load()` samples and assembles +them. Each explicit `load()` can produce a new batch. Retained batches own +their Arrow references after request and loader closure. + +`batch.metadata` and `batch.as_arrow_request()` provide defensive metadata +copies. `batch.tables(split)` returns the instance Arrow table and named +related Arrow tables. `batch.logical_arrow_bytes` sums their `Table.nbytes`; +it is neither peak process memory nor the encoded HTTP body size. Row and +column limits and the decoded Arrow limit apply during assembly. The final +JSON body limit applies only during encoding. Encoding failures leave the +batch usable. + +The wire envelope, composite `(instance_id, primary_key)` identity, selected +`__fm_edge_*` relationship columns, large integer representation, and +context-only target follow the prior loader contract. Arrays remain the +default wire format; Arrow IPC is opt-in. No SDM conversion runs in this wheel. + +## Compatibility + +The production path accepts frozen `native_multihop` workloads whose graph +sampling can be expressed exactly as public `kumo.pytorch` recent sampling. +Uniform sampling into timeless tables and empty graph layers fail explicitly. +The old `local_native` and `temporal_query` selection implementations are +comparison references, not fallbacks in this package. A supplied index must +match the dataset snapshot, contract fingerprint, and recorded dependency +provenance; an unrelated or incomplete index is refused. + +| Contract area | Status | +| --- | --- | +| One or both frozen context/predict splits; ordered instance selection | Supported | +| Classification and regression targets; context label availability checks | Supported | +| Temporal recent sampling, forward/reverse steps, multiple relations and hops | Supported through the public columnar graph API | +| Per-instance identity, selected edges, nulls, empty related tables, exact int64 IDs | Preserved in typed Arrow tables and both encodings | +| Bundle hashes, source schema, keys/FKs, clock precision, rows/columns/bytes | Checked before or during sampling/encoding | +| Uniform sampling into timeless destinations, empty graph layers | Explicitly rejected | +| `local_native` and `temporal_query` selection | Comparison-only; no production fallback | + +| Prior entry point | This package | +| --- | --- | +| `Bundle(root)`, `Workload(bundle, spec)` | `fm.prepare(root, spec, index=...)` | +| `KumoPytorchBackend.prepare_batch(seeds)` | `loader.materialize(context=..., predict=...)` | +| `build_arrow_tables(backend, workload)` | `request.load().as_arrow_request()` | +| `build_payload(..., encoding="arrays")` | `batch.encode("arrays")` | +| `build_payload(..., encoding="arrow_ipc")` | `batch.encode("arrow_ipc")` | +| `arrow_to_sdm(request)` | `examples/local_sdm/adapter.py:arrow_to_sdm(batch)` | + +Source baselines: fm-dataloader `b4516394e98929c5b4752745866d5e3138276255`; +engine `9637e9cc9e2da877959b3c6b9db88c0fc7e35f3f`. The public columnar +API landed in engine PRs #340 and #365. diff --git a/packages/experimental/kumo-fm/examples/local_sdm/README.md b/packages/experimental/kumo-fm/examples/local_sdm/README.md new file mode 100644 index 000000000..cd96be487 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/README.md @@ -0,0 +1,28 @@ +# Optional local SDM example + +This example keeps SDM and its Arrow compatibility handling beside the +notebook. Install matching DiskGraph, `kumo-pytorch`, and `kumo-fm` wheels +from one engine revision, then the pinned SDM environment in +`requirements.lock`. The model may download the +`nvidia/Kumo-Relational` v1.0.1 checkpoint; the prior validation used +snapshot `2bd603d3d8f25f67a7aaa8579908595e20567e22`. + +Set `FM_SDM_FIXTURE` to a directory containing `bundle/` and +`workload.json`. The workload must be an eligible `native_multihop` +contract. Set `FM_SDM_BUNDLE` when the bundle has another path. The notebook opens or creates `fm-index/` there, samples one +batch, converts its retained Arrow tables through public +`TableTensor.from_arrow` and `RelatedTables`, and exercises both model +call forms. The helper only repairs zero-chunk columns and read-only string +buffers when SDM requires it. The same batch is encoded as arrays and IPC. + +The committed smoke case can be recreated with +`python benchmark/fm-payloads/make_fixture.py /tmp/fm-case` from the repository +root. Set `FM_SDM_FIXTURE=/tmp/fm-case` and +`FM_SDM_BUNDLE=/tmp/fm-case/supported` before running `walkthrough.ipynb`. +The notebook includes the local execution output: both model call forms +agreed with a maximum absolute difference of 0.0 on this fixture. + +Direct conversion avoids HTTP serialization; it does not promise that SDM +tensor construction or inference is zero copy. The source baseline is +fm-dataloader PR #27 at +`b4516394e98929c5b4752745866d5e3138276255`. diff --git a/packages/experimental/kumo-fm/examples/local_sdm/adapter.py b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py new file mode 100644 index 000000000..9e7fbdc8a --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py @@ -0,0 +1,68 @@ +"""Example-local conversion from a trusted FM Arrow batch to public SDM inputs.""" +import pyarrow as pa + + +def _compatible_buffers(table): + for index, field in enumerate(table.schema): + column = table.column(index) + if column.num_chunks == 0: + table = table.set_column(index, field, pa.array([], type=field.type)) + elif pa.types.is_string(field.type) or pa.types.is_large_string(field.type): + if any(buffer is not None and not buffer.is_mutable + for chunk in column.chunks for buffer in chunk.buffers()): + table = table.set_column(index, field, pa.concat_arrays(column.chunks)) + return table + + +def arrow_to_sdm(batch): + """Return the one-shot KumoRelational arguments from one retained batch.""" + from sdm import RelatedTables, TableTensor + + payload = batch.as_arrow_request() + if not {"context", "predict"} <= payload.keys(): + raise ValueError("SDM prediction needs both context and predict splits") + schema = payload["schema"] + target = payload["task"]["target"]["column_name"] + target_stype = "numerical" if payload["task"]["kind"] == "regression" else "categorical" + relationships, links = [], [] + for relation in schema["relationships"]: + if relation.get("source_table", "instance_table") == "instance_table": + links.append(dict(task_columns=relation["source_columns"], + table=relation["target_table"], + table_columns=relation["target_columns"])) + else: + relationships.append(dict(left_table=relation["source_table"], + left_columns=relation["source_columns"], + right_table=relation["target_table"], + right_columns=relation["target_columns"])) + instances, related = {}, {} + for split in ("context", "predict"): + tables = {} + entries = {"instance_table": payload[split]["instance_table"], + **payload[split]["related_tables"]} + for name, table in entries.items(): + spec = schema["instance_table"] if name == "instance_table" else schema["related_tables"][name] + stypes = {} + for column in table.column_names: + field = spec["columns"][column] + stype = field.get("stype") + if name == "instance_table" and column == target: + stype = target_stype + elif stype is None: + stype = {"bool": "categorical", "string": "categorical", + "timestamp[us]": "datetime", "date": "datetime"}.get( + field["dtype"], "numerical") + stypes[column] = {"timestamp": "datetime", "id": "id"}.get( + stype.lower(), stype.lower()) + tables[name] = TableTensor.from_arrow( + _compatible_buffers(table), stypes=stypes, device="cpu") + instances[split] = tables.pop("instance_table") + related[split] = RelatedTables( + tables=tables, relationships=relationships, task_links=links) + if target in instances["predict"].stypes: + raise ValueError("Prediction inputs must not contain the target") + return dict(x_context=instances["context"].drop_columns(target), + y_context=instances["context"][[target]], + x_query=instances["predict"], + related_context_tables=related["context"], + related_query_tables=related["predict"]) diff --git a/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock b/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock new file mode 100644 index 000000000..8be1ce059 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/requirements.lock @@ -0,0 +1,3 @@ +pyarrow==24.0.0 +torch==2.13.0 +structured-data-models @ git+https://github.com/NVIDIA/structured-data-models.git@fd2c29d035a165a998ea4fdb2b08a81fd89f71e1 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb new file mode 100644 index 000000000..f7e464471 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb @@ -0,0 +1,123 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "fm-overview", + "metadata": {}, + "source": [ + "# Local FM Arrow to SDM\n", + "\n", + "Provide a validated bundle, workload, compatible DiskGraph index path, and pinned SDM environment described in this folder's README. This notebook samples once and reuses the same Arrow batch for local inference and both HTTP bodies.\n" + ] + }, + { + "cell_type": "code", + "id": "fm-load", + "metadata": {}, + "execution_count": 1, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "instance_id: int64 not null\n", + "__fm_entity_key: int64 not null\n", + "__fm_anchor: timestamp[us, tz=UTC] not null\n", + "__fm_target: string not null\n", + "584\n" + ] + } + ], + "source": [ + "import os\n", + "from pathlib import Path\n", + "import torch\n", + "from kumo import fm\n", + "from adapter import arrow_to_sdm\n", + "from sdm.models import KumoRelational\n", + "\n", + "fixture = Path(os.environ[\"FM_SDM_FIXTURE\"])\n", + "torch.set_num_threads(2)\n", + "bundle_path = Path(os.environ.get(\"FM_SDM_BUNDLE\", str(fixture / \"bundle\")))\n", + "with fm.prepare(bundle_path, fixture / \"workload.json\",\n", + " index=fixture / \"fm-index\", threads=2) as loader:\n", + " with loader.materialize() as request:\n", + " batch = request.load()\n", + "print(batch.tables(\"context\")[\"instance_table\"].schema)\n", + "print(batch.logical_arrow_bytes)\n" + ] + }, + { + "cell_type": "code", + "id": "fm-convert", + "metadata": {}, + "execution_count": 2, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'arrays_bytes': 3374, 'ipc_bytes': 8859}\n" + ] + } + ], + "source": [ + "inputs = arrow_to_sdm(batch)\n", + "arrays_body = batch.encode(\"arrays\")\n", + "ipc_body = batch.encode(\"arrow_ipc\")\n", + "print({\"arrays_bytes\": len(arrays_body), \"ipc_bytes\": len(ipc_body)})\n" + ] + }, + { + "cell_type": "code", + "id": "fm-infer", + "metadata": {}, + "execution_count": 3, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "yes: float\n", + "one-shot/fit-predict max absolute difference: 0.0\n" + ] + } + ], + "source": [ + "model = KumoRelational(task=\"classification\", device=\"cpu\")\n", + "with torch.inference_mode():\n", + " one_shot = model(**inputs, num_estimators=1,\n", + " generator=torch.Generator().manual_seed(42), num_hops=2)\n", + " model.fit(inputs[\"x_context\"], inputs[\"y_context\"],\n", + " inputs[\"related_context_tables\"], num_estimators=1,\n", + " generator=torch.Generator().manual_seed(42), num_hops=2)\n", + " reused_context = model.predict(inputs[\"x_query\"], inputs[\"related_query_tables\"])\n", + "torch.testing.assert_close(one_shot.numerical, reused_context.numerical,\n", + " rtol=1e-5, atol=1e-5)\n", + "print(one_shot.to_arrow().schema)\n", + "print(\"one-shot/fit-predict max absolute difference:\",\n", + " float((one_shot.numerical - reused_context.numerical).abs().max()))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + }, + "fm_validation": { + "result": "passed", + "sdm_revision": "fd2c29d035a165a998ea4fdb2b08a81fd89f71e1", + "model_snapshot": "2bd603d3d8f25f67a7aaa8579908595e20567e22", + "wheel_sha256": "c55014ca6868058f9ddbd4efdcc57032eba24e9152dc445e6e4d95e56edc5818", + "fixture_snapshot": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/packages/experimental/kumo-fm/pyproject.toml b/packages/experimental/kumo-fm/pyproject.toml new file mode 100644 index 000000000..b8be63e92 --- /dev/null +++ b/packages/experimental/kumo-fm/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=77", "pyarrow==24.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "kumo-fm" +version = "0.1.0a1" +description = "FM payload loader over Kumo PyTorch and DiskGraph" +requires-python = ">=3.11" +license = "Apache-2.0" +license-files = ["LICENSE"] +dependencies = [ + "duckdb==2.0.0", + "kumo-pytorch>=0.1.0a1", + "numpy>=1.26,<3", + "pyarrow==24.0.0", +] + +[project.optional-dependencies] +test = ["pytest>=8,<10"] +relbench = ["PyYAML>=6,<7"] + +[tool.uv.sources] +kumo-pytorch = {workspace = true} + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] +include = ["kumo.fm*"] +namespaces = true + +[tool.setuptools.package-data] +"kumo.fm._dataset_support" = ["presets/*.json"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/packages/experimental/kumo-fm/setup.py b/packages/experimental/kumo-fm/setup.py new file mode 100644 index 000000000..dacd71c6f --- /dev/null +++ b/packages/experimental/kumo-fm/setup.py @@ -0,0 +1,8 @@ +import pyarrow +from setuptools import Extension, setup + +setup(ext_modules=[Extension( + "kumo.fm._native_json", ["src/kumo/fm/_native_json.cpp"], + include_dirs=[pyarrow.get_include()], language="c++", + extra_compile_args=["-std=c++17", "-O3"], +)]) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py new file mode 100644 index 000000000..8aa52a40d --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py @@ -0,0 +1,5 @@ +"""FM payload assembly through public Kumo PyTorch columnar batches.""" +from .batch import PayloadBatch +from .loader import FMLoader, PreparedRequest, prepare + +__all__ = ["FMLoader", "PreparedRequest", "PayloadBatch", "prepare"] diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py new file mode 100644 index 000000000..bacad08ba --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py @@ -0,0 +1,145 @@ +"""Validate and assemble selected columnar graph occurrences into FM tables.""" +from collections import defaultdict + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + +from .bundle import TYPES +from ._phase import phase + + +def empty_features(columns): + return pa.table({name: pa.array([], type=TYPES[spec["dtype"]]) for name, spec in columns.items()}) + + +def normalize_features(table, columns): + return pa.table({name: pc.cast(table[name], TYPES[columns[name]["dtype"]]) for name in table.column_names}) + + +def indices(column): + return column.to_numpy(zero_copy_only=False) + + +def _parent_positions(pairs, child_count, relation): + """Canonicalize validated local endpoints without sorting edge pairs. + + A child has at most one selected parent per relation. Scatter all batches + first, then compare *every* original pair with the final map: this catches + conflicts both within and across batches regardless of which repeated + assignment NumPy retains. Local endpoint bounds/ownership are checked by + the caller; -1 denotes an unselected child, not a source primary key. + """ + with phase('native_multihop_parent_map_allocate', 'graph_assembly', relation=relation): + positions = np.full(child_count, -1, dtype=np.int64) + with phase('native_multihop_parent_map_scatter', 'graph_assembly', relation=relation): + for child, parent in pairs: + positions[child] = parent + with phase('native_multihop_parent_map_validate', 'graph_assembly', relation=relation): + for child, parent in pairs: + if not np.array_equal(positions[child], parent): + raise ValueError('Selected child has conflicting parents') + with phase('native_multihop_parent_map_count', 'graph_assembly', relation=relation): + count = int(np.count_nonzero(positions >= 0)) + return positions, count + + +class GraphAssembler: + def __init__(self, bundle): + self.bundle = bundle + self.node_types = list(bundle.tables) + + def _adapt(self, workload, seeds, ids, nodes, features, edge_batches, directions): + """Build final related tables using columnar gathers, not Python edges.""" + from ._contract import schema_for + schema = schema_for(workload)['related_tables'] + frames, owners, node_tables, orders = {}, {}, {}, {} + for name in workload.table_names: + frames[name] = empty_features(self.bundle.tables[name]['columns']) + owners[name] = np.array([], dtype=np.int64) + for type_id in set(nodes) | set(features): + if not nodes[type_id] or not features[type_id]: + raise ValueError('Native fused node/feature alignment mismatch') + name = self.node_types[type_id] + with phase('native_multihop_feature_normalize', 'graph_assembly', table=name): + ns = pa.Table.from_batches(nodes[type_id]) + fs = normalize_features(pa.Table.from_batches(features[type_id]), self.bundle.tables[name]['columns']) + with phase('native_multihop_node_validate', 'graph_assembly', table=name): + if (len(ns) != len(fs) or ns['id_a'].type != pa.uint64() or ns['id_b'].type != pa.uint64() + or ns['id_a'].null_count or ns['id_b'].null_count): + raise ValueError('Native fused node/feature alignment mismatch') + owner = indices(ns['id_a']) + if np.any(owner >= len(seeds)) or np.any(owner < 0): + raise ValueError('Native seed position out of bounds') + if len(fs) > workload.budgets['max_rows_per_table']: + raise ValueError('Expanded related table exceeds row budget') + with phase('native_multihop_identity_sort', 'graph_assembly', table=name): + pk = fs[self.bundle.tables[name]['primary_key']] + identity = pa.table({'i': owner, 'k': pk}) + order = pc.sort_indices(identity, sort_keys=[('i', 'ascending'), ('k', 'ascending')]) + with phase('native_multihop_identity_validate', 'graph_assembly', table=name): + ordered = identity.take(order) + duplicate = pc.and_(pc.equal(ordered['i'].slice(1), ordered['i'].slice(0, max(0, len(fs)-1))), + pc.equal(ordered['k'].slice(1), ordered['k'].slice(0, max(0, len(fs)-1)))) + if pk.null_count or pc.any(duplicate).as_py(): + raise ValueError('Native returned duplicate or null node occurrences') + frames[name], owners[name], node_tables[name], orders[name] = fs, owner, ns, order + with phase('native_multihop_root_validate', 'graph_assembly'): + root = frames[workload.entity] + root_spec = self.bundle.tables[workload.entity] + root_keys = root[root_spec['primary_key']].slice(0, len(seeds)) + if (len(root_keys) != len(seeds) + or not np.array_equal(owners[workload.entity][:len(seeds)], np.arange(len(seeds))) + or not pc.all(pc.fill_null(pc.equal(root_keys, pa.array( + [s['entity_key'] for s in seeds], type=root_keys.type)), False)).as_py()): + raise ValueError('Native root prefix differs from input seeds') + if not np.array_equal(indices(node_tables[workload.entity]['id_b'].slice(0, len(seeds))), np.asarray(ids)): + raise ValueError('Native root IDs differ from resolved seeds') + if root_spec.get('time_column'): + from .bundle import TYPES + anchors = pa.array([s['anchor_time'] for s in seeds], TYPES['timestamp[us]']) + if not pc.all(pc.fill_null(pc.less_equal(root[root_spec['time_column']].slice(0, len(seeds)), anchors), False)).as_py(): + raise ValueError('Task entity is not eligible at its anchor') + + # Positions stay in the engine's original node order until final take(). + pairs = defaultdict(list) + for rel_id, batches in edge_batches.items(): + relation, direction, source, dest = directions[rel_id] + with phase('native_multihop_edge_validate', 'graph_assembly', relation=relation, direction=direction): + batch = pa.Table.from_batches(batches) + if (batch['id_a'].type != pa.uint64() or batch['id_b'].type != pa.uint64() + or batch['id_a'].null_count or batch['id_b'].null_count): + raise ValueError('Native edge has null endpoints') + aa, bb = indices(batch['id_a']), indices(batch['id_b']) + if np.any(aa >= len(frames[source])) or np.any(bb >= len(frames[dest])): + raise ValueError('Native edge position out of bounds') + if not np.array_equal(owners[source][aa], owners[dest][bb]): + raise ValueError('Native edge crosses seed instances') + child, parent = (aa, bb) if direction == 'forward' else (bb, aa) + pairs[relation].append((child, parent)) + edge_columns, edge_count = {}, 0 + for relation in workload.relation_names: + rel = self.bundle.relations[relation] + parent = frames[rel['parent']][self.bundle.tables[rel['parent']]['primary_key']] + positions, count = _parent_positions(pairs[relation], len(frames[rel['child']]), relation) + edge_count += count + with phase('native_multihop_parent_key_gather', 'graph_assembly', relation=relation): + edge_columns[relation] = pc.take(parent, pa.array(positions, mask=positions < 0)) + with phase('native_multihop_instance_ids', 'graph_assembly'): + instance_ids = pa.array([s.get('instance_id', i) for i, s in enumerate(seeds)], pa.int64()) + ready = {} + for name, spec in schema.items(): + columns = {} + for col in spec['columns']: + if col == 'instance_id': + with phase('native_multihop_instance_gather', 'graph_assembly', table=name): + columns[col] = pc.take(instance_ids, pa.array(owners[name])) + elif col.startswith('__fm_edge_'): + columns[col] = edge_columns[col.removeprefix('__fm_edge_')] + else: + columns[col] = frames[name][col] + with phase('native_multihop_table_build', 'graph_assembly', table=name): + table = pa.table(columns) + with phase('native_multihop_output_gather', 'graph_assembly', table=name): + ready[name] = table.take(orders[name]) if name in orders else table + return ready diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py new file mode 100644 index 000000000..a557288da --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py @@ -0,0 +1,57 @@ +"""FM schema and task metadata shared by direct and wire output.""" + +def schema_for(workload): + bundle = workload.bundle + id_spec = {"dtype": "int64", "stype": "ID", "nullable": False} + entity = bundle.tables[workload.entity] + key_spec = {"dtype": entity["columns"][entity["primary_key"]]["dtype"], "stype": "ID", "nullable": False} + instance = {"primary_key": "instance_id", "columns": { + "instance_id": id_spec, "__fm_entity_key": key_spec, + "__fm_anchor": {"dtype": "timestamp[us]", "stype": "timestamp", "nullable": False}, + "__fm_target": {"dtype": workload.target["dtype"] if workload.kind == "regression" else "string", "nullable": False}}} + related = {} + for name in sorted(workload.table_names): + table = bundle.tables[name] + pk = table["primary_key"] + columns = {"instance_id": id_spec, + pk: {"dtype": table["columns"][pk]["dtype"], "stype": "ID", "nullable": False}} + fks = {r["foreign_key"] for r in bundle.relations.values() if r["child"] == name} + for col in workload.projections[name]: + spec = dict(table["columns"][col]) + if spec["dtype"] == "date": + spec["dtype"] = "timestamp[us]" + if col == pk or col in fks: + spec["stype"] = "ID" + columns[col] = spec + related[name] = {"primary_key": ["instance_id", pk], "columns": columns} + relationships = [{"source_columns": ["instance_id", "__fm_entity_key"], + "target_table": workload.entity, + "target_columns": ["instance_id", entity["primary_key"]]}] + for name in sorted(workload.relation_names): + rel = bundle.relations[name] + parent = bundle.tables[rel["parent"]] + dtype = parent["columns"][parent["primary_key"]]["dtype"] + col = "__fm_edge_" + name + related[rel["child"]]["columns"][col] = {"dtype": dtype, "stype": "ID", "nullable": True} + relationships.append({"source_table": rel["child"], "source_columns": ["instance_id", col], + "target_table": rel["parent"], "target_columns": ["instance_id", parent["primary_key"]]}) + return {"instance_table": instance, "related_tables": related, "relationships": relationships} + + + +def metadata_for(workload): + target = dict(workload.target) + if workload.kind != "regression": + target.update(dtype="string", classes=[str(value) for value in target["classes"]]) + fields = ["prediction"] if workload.kind == "regression" else ["prediction", "probabilities"] + return { + "model": "kumo-relational", + "task": { + "kind": workload.kind, + "entity_table_names": [workload.entity], + "anchor_time_column": "__fm_anchor", + "target": {"table_name": "instance_table", "column_name": "__fm_target", **target}, + }, + "schema": schema_for(workload), + "output": {"fields": fields, "score_format": "object"}, + } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py new file mode 100644 index 000000000..3c987464f --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py @@ -0,0 +1,3 @@ +"""Portable, deterministic synthetic relational Parquet bundles.""" + +__version__ = "1.1.0" diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py new file mode 100644 index 000000000..6513f2015 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py @@ -0,0 +1,280 @@ +"""Resolve small declarative presets; reject unsupported knobs before writing.""" + +import json +import math +import re +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from importlib.resources import files + +from . import __version__ + +ALIASES = dict( + zip( + [ + "v_marketo_lead", + "engagements_v2", + "marketing_kumo_bronze_rpt_les_metrics_marketo_agg", + "marketing_kumo_bronze_rpt_les_metrics_marketo_usecase_agg", + "marketing_kumo_bronze_v_marketo_u2_conversions", + "marketing_kumo_bronze_v_test_marketo_trigger_activities", + "v_qual_notes", + ], + [ + "leads", + "engagements", + "metrics", + "usecase_metrics", + "conversions", + "activities", + "notes", + ], + ) +) + + +@dataclass +class Config: + preset: str = "commerce_v1" + scale: float = 0.001 + seed: int = 42 + rows: dict = field(default_factory=dict) + # Overrides keyed by table.column; omitted Marketo relations retain legacy ratios. + relations: dict = field(default_factory=dict) + numeric_features: int = 0 + categorical_features: int = 0 + text_features: int = 0 + category_cardinality: int = 32 + text_length: int = 64 + feature_null_fraction: float = 0.0 + history_start: str = "2023-01-01" + history_days: int = 1095 + recent_share: float = 0.0 + batch_rows: int = 65536 + file_rows: int = 1000000 + row_group_rows: int = 100000 + context_rows: int = 64 + prediction_rows: int = 256 # per degree cohort + lookback_days: int = 365 + threads: int = 2 + memory_limit: str = "2GB" + max_output_gib: float = 8.0 + min_free_gib: float = 5.0 + + def resolve(self): + if self.preset not in ("marketo_v1", "commerce_v1"): + raise ValueError("preset must be marketo_v1 or commerce_v1") + if not math.isfinite(self.scale) or self.scale <= 0: + raise ValueError("scale must be finite and positive") + if not isinstance(self.seed, int) or not 0 <= self.seed < 2**64: + raise ValueError("seed must be an unsigned 64-bit integer") + for name in ( + "batch_rows", + "file_rows", + "row_group_rows", + "context_rows", + "prediction_rows", + "category_cardinality", + "text_length", + "history_days", + "lookback_days", + "threads", + ): + if not isinstance(getattr(self, name), int) or getattr(self, name) < 1: + raise ValueError(f"{name} must be a positive integer") + if self.history_days < 120: + raise ValueError( + "history_days must be at least 120 for complete context/predict labels" + ) + for name in ("numeric_features", "categorical_features", "text_features"): + if not isinstance(getattr(self, name), int) or getattr(self, name) < 0: + raise ValueError(f"{name} must be a nonnegative integer") + for name in ("feature_null_fraction", "recent_share"): + if ( + not math.isfinite(getattr(self, name)) + or not 0 <= getattr(self, name) <= 1 + ): + raise ValueError(f"{name} must be in [0, 1]") + for name in ("max_output_gib", "min_free_gib"): + if not math.isfinite(getattr(self, name)) or getattr(self, name) < 0: + raise ValueError(f"{name} must be finite and nonnegative") + if self.max_output_gib == 0: + raise ValueError("max_output_gib must be positive") + if not re.fullmatch(r"[1-9][0-9]*(MB|GB)", self.memory_limit): + raise ValueError("memory_limit must be an integer number of MB or GB") + start = datetime.strptime(self.history_start, "%Y-%m-%d").replace(tzinfo=UTC) + tables = marketo_tables() if self.preset == "marketo_v1" else commerce_tables() + if set(self.rows) - set(tables): + raise ValueError( + f"Unknown row-count tables: {set(self.rows) - set(tables)}" + ) + for name, table in tables.items(): + n = self.rows.get(name, max(1, round(table["base_rows"] * self.scale))) + if ( + not isinstance(n, int) + or n < (0 if table["time_column"] else 1) + or n > 2**63 - 1 + ): + raise ValueError(f"Invalid row count for {name}: {n}") + table["rows"] = n + for kind, count, dtype in [ + ("numeric", self.numeric_features, "float64"), + ("category", self.categorical_features, "string"), + ("text", self.text_features, "string"), + ]: + for i in range(count): + table["columns"].append( + {"name": f"extra_{kind}_{i}", "type": dtype, "role": kind} + ) + known = set() + for name, table in tables.items(): + for rel in table["fkeys"]: + key = f"{name}.{rel['column']}" + known.add(key) + override = self.relations.get(key, {}) + unknown = set(override) - { + "active_fraction", + "null_fraction", + "distribution", + "hot_fraction", + "hot_share", + "power", + } + if unknown: + raise ValueError(f"Unknown relation options for {key}: {unknown}") + rel.update(override) + for k in ( + "active_fraction", + "null_fraction", + "hot_fraction", + "hot_share", + ): + if not math.isfinite(rel[k]) or not 0 <= rel[k] <= 1: + raise ValueError(f"{key}.{k} must be in [0, 1]") + if rel["distribution"] not in ("uniform", "hotspot", "power"): + raise ValueError(f"Unknown distribution for {key}") + if not math.isfinite(rel["power"]) or rel["power"] <= 0: + raise ValueError("power must be finite and positive") + parents = tables[rel["table"]]["rows"] + linked = round(table["rows"] * (1 - rel["null_fraction"])) + active = ( + min(parents, max(1, round(parents * rel["active_fraction"]))) + if linked + else 0 + ) + # At tiny scales the desired source ratios can be infeasible. Record actual coverage. + active = min(active, linked) + if linked and rel["active_fraction"] == 0: + raise ValueError(f"{key}: linked rows require active parents") + rel.update(linked_rows=linked, active_parents=active) + if set(self.relations) - known: + raise ValueError(f"Unknown relations: {set(self.relations) - known}") + return { + "generator_version": __version__, + "timestamp_granularity": "seconds", + "config": asdict(self), + "tables": tables, + "history_start_us": int(start.timestamp()) * 1000000, + "entity_table": "leads" if self.preset == "marketo_v1" else "users", + "label_table": "engagements" + if self.preset == "marketo_v1" + else "interactions", + } + + +def relation(column, table, key, *, active=1.0, null=0.0, distribution="hotspot"): + return { + "column": column, + "table": table, + "key": key, + "active_fraction": active, + "null_fraction": null, + "distribution": distribution, + "hot_fraction": 0.05, + "hot_share": 0.8, + "power": 3.0, + } + + +def marketo_tables(): + spec = json.loads(files("kumo.fm._dataset_support").joinpath("presets/marketo.json").read_text()) + result = {} + for table in spec["tables"]: + name = ALIASES[table["name"]] + g = table.get("graph") + columns = table["columns"] + if g: + columns = [ + {"name": "_row_id", "type": "int64", "role": "primary_key"}, + *columns, + ] + result[name] = { + "base_rows": g["num_src_nodes"] if g else table["rows"], + "primary_key": "_row_id" if g else "Id", + "time_column": table["time_column"], + "columns": columns, + "fkeys": [ + relation( + next(c["name"] for c in columns if c["role"] == "lead_foreign_key"), + "leads", + "Id", + active=g["num_dst_nonzero_degree"] / spec["lead_rows"], + null=1 - g["edge_count"] / g["num_src_nodes"], + distribution="power", + ) + ] + if g + else [], + } + # Stable schema ordering, independent of the source SPEC's table order. + return {name: result[name] for name in ALIASES.values()} + + +def commerce_tables(): + def col(name, dtype, role): + return {"name": name, "type": dtype, "role": role} + + def table(n, key, columns, fkeys=(), time=None): + return { + "base_rows": n, + "primary_key": key, + "time_column": time, + "columns": [col(key, "int64", "primary_key"), *columns], + "fkeys": list(fkeys), + } + + return { + "users": table(6000000, "user_id", [col("region", "string", "categorical")]), + "interactions": table( + 600000000, + "interaction_id", + [ + col("user_id", "int64", "foreign_key"), + col("item_id", "int64", "foreign_key"), + col("event_time", "time", "timestamp"), + col("amount", "float64", "numeric"), + col("event_type", "string", "categorical"), + ], + [ + relation("user_id", "users", "user_id"), + relation("item_id", "items", "item_id"), + ], + "event_time", + ), + "items": table( + 1000000, + "item_id", + [ + col("category_id", "int64", "foreign_key"), + col("price", "float64", "numeric"), + ], + [ + relation( + "category_id", "categories", "category_id", distribution="uniform" + ) + ], + ), + "categories": table( + 10000, "category_id", [col("name", "string", "categorical")] + ), + } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py new file mode 100644 index 000000000..de8c2c560 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py @@ -0,0 +1,335 @@ +"""Stateless vectorized generation and bounded Parquet writing.""" + +import hashlib +import json +import math +import shutil +import time +from datetime import UTC, datetime +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .storage import ColumnDigests, canonical_hash, file_digest, write_json + +DAY_SECONDS = 86400 +DAY_US = DAY_SECONDS * 1000000 +TYPES = { + "int64": pa.int64(), + "float64": pa.float64(), + "bool": pa.bool_(), + "string": pa.string(), + "time": pa.timestamp("us", tz="UTC"), + "date": pa.date32(), +} + + +def salt(text): + return int.from_bytes(hashlib.sha256(text.encode()).digest()[:8], "little") + + +def random_bits(ids, key): + """SplitMix64 with wraparound, keyed by a stable column/relation namespace.""" + with np.errstate(over="ignore"): + x = np.asarray(ids, dtype=np.uint64) + np.uint64(key) + x = (x ^ (x >> np.uint64(30))) * np.uint64(0xBF58476D1CE4E5B9) + x = (x ^ (x >> np.uint64(27))) * np.uint64(0x94D049BB133111EB) + return x ^ (x >> np.uint64(31)) + + +def uniform(ids, key): + return (random_bits(ids, key) >> np.uint64(11)).astype(np.float64) / 2**53 + + +def prefix(values, text): + return pc.binary_join_element_wise( + pa.scalar(text), pc.cast(pa.array(values), pa.string()), "" + ) + + +def lead_keys(ids): + return pc.binary_join_element_wise( + "L", pc.utf8_lpad(pc.cast(pa.array(ids), pa.string()), 12, "0"), "" + ) + + +def schema_for(table, config): + fkeys = {r["column"]: r for r in table["fkeys"]} + fields = [] + for col in table["columns"]: + nullable = ( + fkeys[col["name"]]["null_fraction"] > 0 + if col["name"] in fkeys + else col["role"] not in ("primary_key", "timestamp") + and config.feature_null_fraction > 0 + ) + fields.append(pa.field(col["name"], TYPES[col["type"]], nullable=nullable)) + return pa.schema(fields) + + +def fk_values(ids, relation, parent_rows, key): + active = relation["active_parents"] + linked = ids < relation["linked_rows"] + if not active: + return np.zeros(len(ids), dtype=np.int64), ~linked + u = uniform(ids, key) + if relation["distribution"] == "power": + tail = np.floor(u ** relation["power"] * active).astype(np.int64) + elif relation["distribution"] == "hotspot": + hot = max(1, min(active, round(active * relation["hot_fraction"]))) + choices = random_bits(ids, key ^ salt("choices")) + if hot == active: + tail = (choices % np.uint64(active)).astype(np.int64) + else: + tail = np.where( + u < relation["hot_share"], + choices % np.uint64(hot), + hot + choices % np.uint64(active - hot), + ).astype(np.int64) + else: + tail = (random_bits(ids, key) % np.uint64(active)).astype(np.int64) + ordinal = np.where(ids < active, ids, tail) + # A bijection over all parents spreads active parents without collisions. + # Keep multiplication within int64, even for a very large parent population. + # Selecting a small coprime multiplier avoids object arrays in the hot path. + multiplier = 65537 + while math.gcd(multiplier, parent_rows) != 1: + multiplier += 2 + if parent_rows > (2**63 - 1) // multiplier: + raise ValueError("Parent population too large for the generator key mapping") + values = (ordinal * multiplier + key % parent_rows) % parent_rows + return values, ~linked + + +def batch(recipe, name, start, size): + config = Config(**recipe["config"]) + table = recipe["tables"][name] + ids = np.arange(start, start + size, dtype=np.int64) + namespace = config.seed ^ salt(name) + fkeys = {} + for rel in table["fkeys"]: + fkeys[rel["column"]] = fk_values( + ids, + rel, + recipe["tables"][rel["table"]]["rows"], + namespace ^ salt(rel["column"]), + ) + event = random_bits(ids, namespace ^ salt("event-time")) % np.uint64( + config.history_days * DAY_SECONDS + ) + recent = uniform(ids, namespace ^ salt("recent")) < config.recent_share + recent_days = min(90, config.history_days) + event = np.where( + recent, + (config.history_days - recent_days) * DAY_SECONDS + + event % np.uint64(recent_days * DAY_SECONDS), + event, + ).astype(np.int64) + # Draw whole-second values; retain timestamp[us] as the physical schema. + event = event * 1000000 + recipe["history_start_us"] + arrays = [] + for col in table["columns"]: + cname, dtype, role = col["name"], col["type"], col["role"] + key = namespace ^ salt(cname) + nulls = None + if role == "primary_key": + value = lead_keys(ids) if dtype == "string" else pa.array(ids) + elif cname in fkeys: + values, nulls = fkeys[cname] + value = lead_keys(values) if dtype == "string" else pa.array(values) + elif dtype == "time": + value = pa.array(event, type=TYPES[dtype]) + elif dtype == "date": + days = event // DAY_US + if cname.startswith("first_"): + days = days - (random_bits(ids, key) % 365).astype(np.int64) - 1 + value = pa.array(days.astype(np.int32), type=pa.date32()) + elif dtype == "bool": + value = pa.array(uniform(ids, key) < (0.2 if cname == "is_mql" else 0.05)) + elif dtype == "int64": + # Preserve cumulative Marketo window shapes without Spark dependencies. + root = ( + cname.rsplit("_", 1)[0] + if cname.endswith(("_30", "_60", "_90")) + else cname + ) + terms = [ + random_bits(ids, namespace ^ salt(root + suffix)) % modulus + for suffix, modulus in [("a", 5), ("b", 4), ("c", 3), ("d", 6)] + ] + count = {"30": 1, "60": 2, "90": 3}.get(cname.rsplit("_", 1)[-1], 4) + values = sum(terms[:count]).astype(np.int64) + if cname.startswith("days_since"): + values = (random_bits(ids, key) % 366).astype(np.int64) + value = pa.array(values) + elif dtype == "float64": + value = pa.array(uniform(ids, key) * 1000) + elif role == "text": + # Bounded-cardinality text; lengths/cardinality are explicit compression controls. + values = random_bits(ids, key) % config.category_cardinality + value = pc.utf8_slice_codeunits( + pc.utf8_rpad(prefix(values, "text_"), config.text_length, "x"), + 0, + config.text_length, + ) + else: + identity = ( + fkeys.get("marketo_lead_id", (ids, None))[0] + if cname.startswith(("account_", "audience_")) + else ids + ) + cardinality = config.category_cardinality + if config.preset == "marketo_v1" and not cname.startswith("extra_"): + cardinality = ( + 250000 + if cname in ("account_id", "account_name", "company") + else ( + 5000 if cname in ("tactic_name", "tactic_program_name") else 12 + ) + ) + value = prefix(random_bits(identity, key) % cardinality, cname + "_") + if role not in ("primary_key", "timestamp") and cname not in fkeys: + nulls = uniform(ids, key ^ salt("nulls")) < config.feature_null_fraction + if nulls is not None: + value = pc.if_else(pa.array(nulls), pa.scalar(None, type=value.type), value) + arrays.append(value) + return pa.RecordBatch.from_arrays(arrays, schema=schema_for(table, config)) + + +class Budget: + def __init__(self, root, config): + self.root, self.config = root, config + + def check(self): + size = sum(p.stat().st_size for p in self.root.rglob("*") if p.is_file()) + free = shutil.disk_usage(self.root).free + if size > self.config.max_output_gib * 2**30: + raise RuntimeError( + "Output disk budget exceeded; incomplete bundle retained for inspection" + ) + if free < self.config.min_free_gib * 2**30: + raise RuntimeError( + "Free-space reserve reached; incomplete bundle retained for inspection" + ) + return size + + +def plan(config): + recipe = config.resolve() + # An uncompressed estimate, not a promised compressed footprint or peak-RSS bound. + estimated = 0 + for table in recipe["tables"].values(): + for col in table["columns"]: + estimated += table["rows"] * ( + config.text_length + 8 + if col["role"] == "text" + else 40 + if col["type"] == "string" + else 9 + ) + return { + "preset": config.preset, + "rows": {k: v["rows"] for k, v in recipe["tables"].items()}, + "edges": sum( + r["linked_rows"] for t in recipe["tables"].values() for r in t["fkeys"] + ), + "estimated_uncompressed_bytes": estimated, + "recipe": recipe, + } + + +def generate(root: Path, config: Config): + from .tasks import build_tasks + from .validate import validate + + root = root.resolve() + recipe = config.resolve() # Fail before creating output for invalid configurations. + root.mkdir(parents=True, exist_ok=False) + budget = Budget(root, config) + budget.check() + started = time.monotonic() + manifest = { + "format": "fm-dataset-v1", + "recipe": recipe, + "tables": {}, + "tasks": {}, + "created_utc": datetime.now(UTC).isoformat(), + "versions": {"numpy": np.__version__, "pyarrow": pa.__version__}, + } + for name, table in recipe["tables"].items(): + directory = root / "tables" / name + directory.mkdir(parents=True) + schema = schema_for(table, config) + digests = ColumnDigests(schema) + inventory = [] + for shard, file_start in enumerate( + range(0, max(1, table["rows"]), config.file_rows) + ): + path = directory / f"part-{shard:05d}.parquet" + stop = min(file_start + config.file_rows, table["rows"]) + with pq.ParquetWriter(path, schema, compression="zstd") as writer: + for group_start in range(file_start, stop, config.row_group_rows): + group_stop = min(stop, group_start + config.row_group_rows) + chunks = [] + for start in range(group_start, group_stop, config.batch_rows): + chunk = batch( + recipe, + name, + start, + min(config.batch_rows, group_stop - start), + ) + digests.update(chunk) + chunks.append(chunk) + writer.write_table( + pa.Table.from_batches(chunks), + row_group_size=group_stop - group_start, + ) + budget.check() + meta = pq.ParquetFile(path).metadata + inventory.append( + { + "path": path.relative_to(root).as_posix(), + "bytes": path.stat().st_size, + "sha256": file_digest(path), + "row_groups": [ + meta.row_group(i).num_rows for i in range(meta.num_row_groups) + ], + } + ) + budget.check() + manifest["tables"][name] = { + "rows": table["rows"], + "schema": schema.to_string(), + "logical_sha256": digests.hexdigest(), + "files": inventory, + } + print( + json.dumps({"table": name, "rows": table["rows"], "files": len(inventory)}), + flush=True, + ) + manifest["logical_id"] = canonical_hash( + { + name: {k: v[k] for k in ("rows", "schema", "logical_sha256")} + for name, v in manifest["tables"].items() + } + ) + manifest["snapshot_id"] = canonical_hash(manifest["tables"]) + write_json(root / "manifest.json", manifest) + build_tasks(root, manifest, budget) + manifest["generation_seconds"] = time.monotonic() - started + write_json(root / "manifest.json", manifest) + report = validate(root, allow_incomplete=True) + write_json(root / "validation.json", report) + budget.check() + write_json( + root / "_COMPLETE.json", + { + "manifest_sha256": file_digest(root / "manifest.json"), + "validation_sha256": file_digest(root / "validation.json"), + }, + ) + return manifest diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json new file mode 100644 index 000000000..07f4553da --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/presets/marketo.json @@ -0,0 +1,504 @@ +{ + "description": "Synthetic version of the supplied lead-centered graph; source table counts and exact job edge stats.", + "graph_leads": 6889362, + "lead_rows": 6923193, + "default_history_start": "2023-01-01", + "default_history_days": 1095, + "tables": [ + { + "name": "v_qual_notes", + "rows": 81751, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "source_creation_timestamp", + "type": "time", + "role": "timestamp" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + } + ], + "time_column": "source_creation_timestamp", + "graph": { + "edge_count": 77474, + "num_src_nodes": 81446, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 61107 + } + }, + { + "name": "engagements_v2", + "rows": 437528829, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "engagement_datetime", + "type": "time", + "role": "timestamp" + }, + { + "name": "engagement_type", + "type": "string", + "role": "categorical" + }, + { + "name": "engagement_event_asset", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_title", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_persona", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_region", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_sub_region", + "type": "string", + "role": "categorical" + }, + { + "name": "audience_country", + "type": "string", + "role": "categorical" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + }, + { + "name": "account_name", + "type": "string", + "role": "categorical" + }, + { + "name": "account_dbx", + "type": "string", + "role": "categorical" + }, + { + "name": "account_company_size", + "type": "string", + "role": "categorical" + }, + { + "name": "account_industry", + "type": "string", + "role": "categorical" + }, + { + "name": "account_vertical", + "type": "string", + "role": "categorical" + }, + { + "name": "account_sales_region", + "type": "string", + "role": "categorical" + }, + { + "name": "account_business_unit", + "type": "string", + "role": "categorical" + }, + { + "name": "account_spend_tier", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_name", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_program_name", + "type": "string", + "role": "categorical" + }, + { + "name": "tactic_product_focus", + "type": "string", + "role": "categorical" + } + ], + "time_column": "engagement_datetime", + "graph": { + "edge_count": 378563179, + "num_src_nodes": 435519698, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 4724442 + } + }, + { + "name": "v_marketo_lead", + "rows": 6923193, + "columns": [ + { + "name": "Id", + "type": "string", + "role": "primary_key" + } + ], + "time_column": null + }, + { + "name": "marketing_kumo_bronze_v_marketo_u2_conversions", + "rows": 609072, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "u2_stage_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "account_id", + "type": "string", + "role": "ID" + }, + { + "name": "audience_role", + "type": "string", + "role": "categorical" + }, + { + "name": "seniority_level", + "type": "string", + "role": "categorical" + }, + { + "name": "company", + "type": "string", + "role": "categorical" + }, + { + "name": "account_name", + "type": "string", + "role": "categorical" + } + ], + "time_column": "u2_stage_date", + "graph": { + "edge_count": 384578, + "num_src_nodes": 605830, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 139119 + } + }, + { + "name": "marketing_kumo_bronze_v_test_marketo_trigger_activities", + "rows": 20918304, + "columns": [ + { + "name": "id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "activity_date", + "type": "time", + "role": "timestamp" + } + ], + "time_column": "activity_date", + "graph": { + "edge_count": 14773740, + "num_src_nodes": 20841579, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 4041373 + } + }, + { + "name": "marketing_kumo_bronze_rpt_les_metrics_marketo_agg", + "rows": 180883041, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "engagement_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "engagement_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_engagement", + "type": "int64", + "role": "categorical" + }, + { + "name": "is_nnn", + "type": "bool", + "role": "categorical" + }, + { + "name": "is_mql", + "type": "bool", + "role": "categorical" + }, + { + "name": "is_qual_notes", + "type": "bool", + "role": "categorical" + }, + { + "name": "first_nnn_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "first_mql_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "first_qual_notes_date", + "type": "date", + "role": "timestamp" + }, + { + "name": "days_since_last_nnn", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_mql", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_qual_notes", + "type": "int64", + "role": "categorical" + }, + { + "name": "registration_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "registration_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "attendance_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_registration", + "type": "int64", + "role": "categorical" + }, + { + "name": "days_since_last_attendance", + "type": "int64", + "role": "categorical" + }, + { + "name": "trigger_date", + "type": "date", + "role": "timestamp" + } + ], + "time_column": "trigger_date", + "graph": { + "edge_count": 133735759, + "num_src_nodes": 180818601, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 5078899 + } + }, + { + "name": "marketing_kumo_bronze_rpt_les_metrics_marketo_usecase_agg", + "rows": 9281224, + "columns": [ + { + "name": "marketo_lead_id", + "type": "string", + "role": "lead_foreign_key" + }, + { + "name": "associated_u1_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u1_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u1_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u2_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u2_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u2_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u3_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u3_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u3_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "associated_u6_count", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_30", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_60", + "type": "int64", + "role": "numerical" + }, + { + "name": "associated_u6_count_90", + "type": "int64", + "role": "numerical" + }, + { + "name": "days_since_last_u6_association", + "type": "int64", + "role": "categorical" + }, + { + "name": "trigger_date", + "type": "date", + "role": "timestamp" + } + ], + "time_column": "trigger_date", + "graph": { + "edge_count": 5855118, + "num_src_nodes": 9274298, + "num_dst_nodes": 6889362, + "num_dst_nonzero_degree": 115612 + } + } + ] +} diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py new file mode 100644 index 000000000..7578a2c26 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py @@ -0,0 +1,603 @@ +"""Pinned, streaming adapters for the original RelBench F1 and H&M databases.""" + +import json +import re +import shutil +import urllib.request +from dataclasses import asdict +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .generate import TYPES, Budget +from .storage import ( + ColumnDigests, + canonical_hash, + file_digest, + quote, + safe_path, + write_json, +) + +REPOSITORY = "stanford-star/relbench-v1" +REVISION = "21d7284df2a9621c71f21bc4a1a30b7e8effb8dd" +TASKS = { + "rel-f1": {"driver-dnf": 30, "driver-position": 60}, + "rel-hm": {"user-churn": 7}, +} +ROW_ID = "_fm_source_row_id" + + +def yaml_read(path): + try: + import yaml + except ImportError as exc: + raise RuntimeError( + "RelBench imports require PyYAML: install fm-dataloader[relbench]" + ) from exc + + return yaml.safe_load(path.read_text()) + + +def fetch(cache, relative, budget): + """Reuse only checksum-verified files under the immutable revision namespace.""" + path = safe_path(cache, relative) + receipt = path.with_suffix(path.suffix + ".sha256") + if path.exists() and receipt.exists(): + if file_digest(path) != receipt.read_text().strip(): + raise ValueError(f"Corrupt cached source: {path}") + return path + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".partial") + url = f"https://huggingface.co/datasets/{REPOSITORY}/resolve/{REVISION}/{relative}" + with ( + urllib.request.urlopen(url, timeout=120) as response, + temporary.open("wb") as stream, + ): + while chunk := response.read(8 * 2**20): + stream.write(chunk) + budget.check() + temporary.replace(path) + receipt.write_text(file_digest(path) + "\n") + return path + + +def normalized_type(dtype): + if pa.types.is_dictionary(dtype): + return normalized_type(dtype.value_type) + if pa.types.is_timestamp(dtype): + return pa.timestamp("us", tz="UTC") + if pa.types.is_integer(dtype): + return pa.int64() + if pa.types.is_floating(dtype): + return pa.float64() + if pa.types.is_string(dtype) or pa.types.is_large_string(dtype): + return pa.string() + if pa.types.is_boolean(dtype) or pa.types.is_date32(dtype): + return dtype + raise ValueError(f"Unsupported source type: {dtype}") + + +def imported_schema(table): + return pa.schema( + [pa.field(c["name"], TYPES[c["type"]], c["nullable"]) for c in table["columns"]] + ) + + +def normalize(batch, schema, offset=0): + arrays = [] + for field in schema: + if field.name == ROW_ID: + arrays.append(pa.array(np.arange(offset, offset + len(batch)), pa.int64())) + else: + # safe=True rejects precision loss (including sub-microsecond timestamps). + arrays.append( + pc.cast( + batch.column(batch.schema.get_field_index(field.name)), + field.type, + safe=True, + ) + ) + return pa.RecordBatch.from_arrays(arrays, schema=schema) + + +def artifact(root, path, rows=None): + result = {"path": path.relative_to(root).as_posix(), "sha256": file_digest(path)} + if rows is not None: + result["rows"] = rows + return result + + +def import_relbench(root, cache, dataset, config=None): + """Import the complete database; never independently sample relational tables.""" + from .validate import validate + + if dataset not in TASKS: + raise ValueError(f"Unsupported dataset: {dataset}") + cfg = config or Config() + # Validate resource controls using the existing configuration validator. + cfg.resolve() + if cfg.context_rows < 1 or cfg.prediction_rows < 1: + raise ValueError("Task sample sizes must be positive") + root, cache = Path(root).resolve(), Path(cache).resolve() / REVISION + if root.exists(): + raise FileExistsError(f"Refusing to overwrite {root}") + cache.mkdir(parents=True, exist_ok=True) + source_budget = Budget(cache, cfg) + source_budget.check() + def get(relative): + return fetch(cache, relative, source_budget) + source_manifest = get(f"{dataset}/manifest.yaml") + meta = yaml_read(source_manifest) + if meta["name"] != dataset or meta["manifest_version"] != 1: + raise ValueError("Unexpected upstream dataset manifest") + root.mkdir(parents=True) + budget = Budget(root, cfg) + source_dir = root / "source" + source_dir.mkdir() + provenance = [] + for relative in ["README.md", f"{dataset}/README.md", f"{dataset}/manifest.yaml"]: + original = get(relative) + destination = source_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(original, destination) + provenance.append(artifact(root, destination)) + recipe = { + "source_kind": "relbench", + "adapter_version": 1, + "config": asdict(cfg), + "dataset": dataset, + "tables": {}, + "repository": REPOSITORY, + "revision": REVISION, + "val_timestamp": meta["val_timestamp"], + "test_timestamp": meta["test_timestamp"], + } + manifest = { + "format": "fm-dataset-v1", + "recipe": recipe, + "tables": {}, + "tasks": {}, + "workloads": [], + "source_artifacts": provenance, + "versions": {"pyarrow": pa.__version__}, + "created_utc": datetime.now(UTC).isoformat(), + } + type_names = {v: k for k, v in TYPES.items()} + for name, spec in meta["tables"].items(): + source = get(f"{dataset}/db/{name}.parquet") + pf = pq.ParquetFile(source) + key = spec["pkey"] or ROW_ID + if ROW_ID in pf.schema_arrow.names: + raise ValueError("Reserved occurrence-ID column already exists") + fields = [ + pa.field(f.name, normalized_type(f.type), f.name != key) + for f in pf.schema_arrow + ] + if spec["pkey"] is None: + fields.insert(0, pa.field(ROW_ID, pa.int64(), False)) + schema = pa.schema(fields) + table = { + "rows": pf.metadata.num_rows, + "primary_key": key, + "time_column": spec["time_col"], + "key_policy": "source-key" + if spec["pkey"] + else "zero-based-pinned-source-row-offset", + "source": { + "path": f"{dataset}/db/{name}.parquet", + "sha256": file_digest(source), + "schema": pf.schema_arrow.remove_metadata().to_string(), + }, + "columns": [ + { + "name": f.name, + "type": type_names[f.type], + "nullable": f.nullable, + "role": "primary_key" + if f.name == key + else "timestamp" + if f.name == spec["time_col"] + else "feature", + } + for f in schema + ], + "fkeys": [ + {"column": col, "table": parent, "key": meta["tables"][parent]["pkey"]} + for col, parent in spec["fkeys"].items() + ], + } + recipe["tables"][name] = table + folder = root / "tables" / name + folder.mkdir(parents=True) + digest, inventory, offset, writer, file_count = ( + ColumnDigests(schema), + [], + 0, + None, + 0, + ) + + def close_part(writer, path, inventory): + writer.close() + m = pq.ParquetFile(path).metadata + inventory.append( + { + **artifact(root, path), + "bytes": path.stat().st_size, + "row_groups": [ + m.row_group(i).num_rows for i in range(m.num_row_groups) + ], + } + ) + + try: + for batch in pf.iter_batches(batch_size=cfg.batch_rows): + chunk = normalize(batch, schema, offset) + digest.update(chunk) + offset += len(chunk) + start = 0 + while start < len(chunk): + if writer is None: + path = folder / f"part-{len(inventory):05d}.parquet" + writer = pq.ParquetWriter(path, schema, compression="zstd") + file_count = 0 + count = min(len(chunk) - start, cfg.file_rows - file_count) + writer.write_batch( + chunk.slice(start, count), row_group_size=cfg.row_group_rows + ) + start += count + file_count += count + if file_count == cfg.file_rows: + close_part(writer, path, inventory) + writer = None + budget.check() + if writer is None and not inventory: + path = folder / "part-00000.parquet" + writer = pq.ParquetWriter(path, schema, compression="zstd") + finally: + if writer is not None: + close_part(writer, path, inventory) + manifest["tables"][name] = { + "rows": offset, + "schema": schema.to_string(), + "logical_sha256": digest.hexdigest(), + "files": inventory, + } + print( + json.dumps({"dataset": dataset, "table": name, "rows": offset}), flush=True + ) + manifest["logical_id"] = canonical_hash( + { + n: {k: t[k] for k in ("rows", "schema", "logical_sha256")} + for n, t in manifest["tables"].items() + } + ) + manifest["snapshot_id"] = canonical_hash(manifest["tables"]) + build_imported_tasks(root, manifest, get, budget) + write_json(root / "manifest.json", manifest) + report = validate(root, allow_incomplete=True) + write_json(root / "validation.json", report) + budget.check() + write_json( + root / "_COMPLETE.json", + { + "manifest_sha256": file_digest(root / "manifest.json"), + "validation_sha256": file_digest(root / "validation.json"), + }, + ) + return manifest + + +def task_rows(path, spec, split, limit=None, cutoff=None): + """Read bounded samples in pinned source order, retaining source row identity.""" + rows, offset = [], 0 + horizon = timedelta(days=spec["horizon_days"]) + for batch in pq.ParquetFile(path).iter_batches(batch_size=65536): + for row in batch.to_pylist(): + row_id = offset + offset += 1 + anchor = row[spec["time_col"]] + anchor = ( + anchor.replace(tzinfo=UTC) + if anchor.tzinfo is None + else anchor.astimezone(UTC) + ) + if cutoff is not None and anchor + horizon > cutoff: + continue + target = row[spec["target_col"]] + rows.append( + { + "instance_id": row_id + + {"train": 0, "val": 10**12, "test": 2 * 10**12}[split], + "entity_key": row[spec["entity_col"]], + "anchor_time": anchor, + "target": target, + "label_available_time": anchor + horizon, + } + ) + if limit is not None and len(rows) == limit: + return rows + return rows + + +def build_imported_tasks(root, manifest, get, budget): + recipe, cfg = manifest["recipe"], manifest["recipe"]["config"] + dataset = recipe["dataset"] + for task, days in TASKS[dataset].items(): + relative = f"{dataset}/tasks/{task}/manifest.yaml" + original = get(relative) + spec = yaml_read(original) + if spec["kind"] != "forecast" or spec["timedelta"] != f"{days} days": + raise ValueError("Unexpected upstream task horizon/kind") + spec["horizon_days"] = days + folder = root / "tasks" / task + folder.mkdir(parents=True) + shutil.copyfile(original, folder / "source-manifest.yaml") + source_spec = artifact(root, folder / "source-manifest.yaml") + splits = {} + for split in ("train", "val", "test"): + source = get(f"{dataset}/tasks/{task}/{split}.parquet") + destination = folder / f"source-{split}.parquet" + shutil.copyfile(source, destination) + splits[split] = artifact( + root, destination, pq.ParquetFile(destination).metadata.num_rows + ) + budget.check() + predictions = task_rows( + root / splits["val"]["path"], spec, "val", cfg["prediction_rows"] + ) + cutoff = min(r["anchor_time"] for r in predictions) + contexts = task_rows( + root / splits["train"]["path"], spec, "train", cfg["context_rows"], cutoff + ) + if not contexts or not predictions: + raise ValueError("No eligible context/prediction instances") + target_type = pa.float64() if spec["task_type"] == "regression" else pa.int64() + key = recipe["tables"][spec["entity_table"]]["primary_key"] + key_type = ( + imported_schema(recipe["tables"][spec["entity_table"]]).field(key).type + ) + schema = pa.schema( + [ + pa.field("instance_id", pa.int64(), False), + pa.field("entity_key", key_type, False), + pa.field("anchor_time", pa.timestamp("us", tz="UTC"), False), + pa.field("target", target_type, False), + pa.field("label_available_time", pa.timestamp("us", tz="UTC"), False), + ] + ) + context, prediction = ( + pa.Table.from_pylist(contexts, schema), + pa.Table.from_pylist(predictions, schema), + ) + artifacts = {} + for kind, data in [ + ("context", context), + ( + "predict", + prediction.select(["instance_id", "entity_key", "anchor_time"]), + ), + ("labels", prediction.select(["instance_id", "target"])), + ]: + path = folder / f"{kind}.parquet" + pq.write_table(data, path, compression="zstd") + artifacts[kind] = artifact(root, path, len(data)) + manifest["tasks"][task] = { + "type": "regression" if target_type == pa.float64() else "classification", + "entity_table": spec["entity_table"], + "label_interval": f"(anchor, anchor + {days} days]", + "source_spec": source_spec, + "source_splits": splits, + "source_metadata": spec, + "prediction_split": "val", + "selection": "first-eligible-pinned-source-rows-v1", + "artifacts": artifacts, + } + for size in sorted( + {min(n, len(prediction)) for n in (1, 32, cfg["prediction_rows"])} + ): + paths = ( + [ + ["drivers", "results", "races", "circuits"], + ["drivers", "results", "constructors"], + ] + if dataset == "rel-f1" + else [["customer", "transactions", "article"]] + ) + work = { + "task": task, + "logical_id": manifest["logical_id"], + "snapshot_id": manifest["snapshot_id"], + "cohort": "official-val", + "context_instance_ids": context["instance_id"].to_pylist(), + "prediction_instance_ids": prediction["instance_id"].to_pylist()[:size], + "paths": paths, + "event_fanout": 16, + "forward_fk_fanout": 1, + "lookback_days": cfg["lookback_days"], + "history_interval": "[anchor-lookback, anchor)", + "event_order": ["timestamp DESC", "primary_key ASC"], + "timeless_dimensions": [ + n for n, t in recipe["tables"].items() if not t["time_column"] + ], + "identity_policy": "deduplicate source IDs within each instance; preserve edges and per-instance occurrences", + "projection": { + n: [c["name"] for c in t["columns"]] + for n, t in recipe["tables"].items() + }, + "feature_columns": { + n: [ + c["name"] + for c in t["columns"] + if c["role"] != "primary_key" + and c["name"] not in {r["column"] for r in t["fkeys"]} + ] + for n, t in recipe["tables"].items() + }, + } + work["workload_id"] = canonical_hash(work) + path = root / "workloads" / f"{task}-official-val-{size}.json" + path.parent.mkdir(exist_ok=True) + write_json(path, work) + manifest["workloads"].append(artifact(root, path)) + + +def validate_recipe(root, manifest): + recipe = manifest["recipe"] + if ( + recipe["repository"] != REPOSITORY + or recipe["revision"] != REVISION + or recipe["adapter_version"] != 1 + ): + raise ValueError("Unrecognized RelBench source pin/adapter") + dataset = recipe["dataset"] + if dataset not in TASKS or set(manifest["tasks"]) != set(TASKS[dataset]): + raise ValueError("Unexpected RelBench dataset/tasks") + for item in manifest["source_artifacts"]: + if file_digest(safe_path(root, item["path"])) != item["sha256"]: + raise ValueError("Source metadata checksum mismatch") + source = yaml_read(root / "source" / dataset / "manifest.yaml") + if set(source["tables"]) != set(recipe["tables"]): + raise ValueError("Source table inventory mismatch") + for name, spec in source["tables"].items(): + table = recipe["tables"][name] + expected = [ + {"column": col, "table": parent, "key": source["tables"][parent]["pkey"]} + for col, parent in spec["fkeys"].items() + ] + if ( + table["primary_key"] != (spec["pkey"] or ROW_ID) + or table["time_column"] != spec["time_col"] + or table["fkeys"] != expected + ): + raise ValueError("Source relational metadata mismatch") + if not re.fullmatch(r"[0-9a-f]{64}", table["source"]["sha256"]): + raise ValueError("Missing source checksum") + for key in ("val_timestamp", "test_timestamp"): + if recipe[key] != source[key]: + raise ValueError("Source split cutoff mismatch") + return recipe + + +def validate_tasks(root, manifest, con, report): + instance_sets = {} + recipe, cfg = manifest["recipe"], manifest["recipe"]["config"] + for task, task_spec in manifest["tasks"].items(): + source_spec = task_spec["source_spec"] + path = safe_path(root, source_spec["path"]) + if file_digest(path) != source_spec["sha256"]: + raise ValueError("Source task metadata checksum mismatch") + spec = yaml_read(path) + spec["horizon_days"] = TASKS[recipe["dataset"]][task] + if ( + spec != task_spec["source_metadata"] + or spec["timedelta"] != f"{spec['horizon_days']} days" + ): + raise ValueError("Source task specification mismatch") + if ( + task_spec["entity_table"] != spec["entity_table"] + or task_spec["prediction_split"] != "val" + ): + raise ValueError("Task mapping mismatch") + if set(task_spec["source_splits"]) != {"train", "val", "test"}: + raise ValueError("Missing official task splits") + for split, item in task_spec["source_splits"].items(): + path = safe_path(root, item["path"]) + pf = pq.ParquetFile(path) + if ( + file_digest(path) != item["sha256"] + or pf.metadata.num_rows != item["rows"] + ): + raise ValueError("Official task split checksum/count mismatch") + con.read_parquet(str(path)).create_view("official_split", replace=True) + entity, col, stamp, target = map( + quote, + [ + spec["entity_table"], + spec["entity_col"], + spec["time_col"], + spec["target_col"], + ], + ) + pk = quote(recipe["tables"][spec["entity_table"]]["primary_key"]) + missing = con.execute( + f"SELECT count(*) FROM official_split s ANTI JOIN {entity} e ON s.{col}=e.{pk}" + ).fetchall()[0][0] + nulls = con.execute( + f"SELECT count(*) FROM official_split WHERE {col} IS NULL OR {stamp} IS NULL OR {target} IS NULL" + ).fetchall()[0][0] + val, test = recipe["val_timestamp"], recipe["test_timestamp"] + condition = ( + f"{stamp} >= ?::TIMESTAMP" + if split == "train" + else f"{stamp} < ?::TIMESTAMP OR {stamp} >= ?::TIMESTAMP" + if split == "val" + else f"{stamp} < ?::TIMESTAMP" + ) + params = ( + [val] if split == "train" else [val, test] if split == "val" else [test] + ) + wrong_split = con.execute( + f"SELECT count(*) FROM official_split WHERE {condition}", params + ).fetchall()[0][0] + if missing or nulls or wrong_split: + raise ValueError( + f"Invalid official split entities/labels/cutoffs: {task}/{split}" + ) + predictions = task_rows( + safe_path(root, task_spec["source_splits"]["val"]["path"]), + spec, + "val", + cfg["prediction_rows"], + ) + cutoff = min(r["anchor_time"] for r in predictions) + contexts = task_rows( + safe_path(root, task_spec["source_splits"]["train"]["path"]), + spec, + "train", + cfg["context_rows"], + cutoff, + ) + expected = { + "context": contexts, + "predict": [ + {k: r[k] for k in ("instance_id", "entity_key", "anchor_time")} + for r in predictions + ], + "labels": [ + {k: r[k] for k in ("instance_id", "target")} for r in predictions + ], + } + if set(task_spec["artifacts"]) != set(expected): + raise ValueError("Missing task artifacts") + for kind, item in task_spec["artifacts"].items(): + path = safe_path(root, item["path"]) + data = pq.read_table(path) + if ( + file_digest(path) != item["sha256"] + or len(data) != item["rows"] + or data.to_pylist() != expected[kind] + ): + raise ValueError( + f"Task differs from official labels/selection: {task}/{kind}" + ) + instance_sets[task] = { + kind: {r["instance_id"] for r in expected[kind]} + for kind in ("context", "predict") + } + report["tasks"][task] = { + "source_splits": { + k: v["rows"] for k, v in task_spec["source_splits"].items() + }, + "context": len(contexts), + "predict": len(predictions), + "horizon_days": spec["horizon_days"], + "labels_verified_against": "pinned official splits (not recomputed)", + } + return instance_sets diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py new file mode 100644 index 000000000..ed2e07510 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py @@ -0,0 +1,259 @@ +"""Batch-independent logical hashes and portable artifact inventories.""" + +import hashlib +import json +import shutil +from pathlib import Path + +import numpy as np +import pyarrow as pa +import pyarrow.compute as pc + + +def canonical_hash(value): + return hashlib.sha256( + json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode() + ).hexdigest() + + +def file_digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def write_json(path, value): + Path(path).write_text( + json.dumps(value, indent=2, sort_keys=True, allow_nan=False) + "\n" + ) + + +def safe_path(root, relative): + path = (root / relative).resolve() + if not path.is_relative_to(root.resolve()): + raise ValueError(f"Artifact path escapes bundle: {relative}") + return path + + +class ColumnDigests: + """Version 1: independent validity/value/length streams in PK source order. + + Separating streams makes the result independent of writer/reader chunk sizes. + Null values use zero/empty bytes; validity distinguishes null from those values. + """ + + def __init__(self, schema): + self.schema = schema + self.streams = [[hashlib.sha256() for _ in range(3)] for _ in schema] + + def update(self, batch): + for field, array, (valid, values, lengths) in zip( + self.schema, batch.columns, self.streams, strict=True + ): + valid.update(np.asarray(pc.is_valid(array)).astype(np.uint8).tobytes()) + if pa.types.is_string(field.type): + filled = pc.fill_null(array, "") + offsets = np.frombuffer(filled.buffers()[1], dtype=" int64 epoch microseconds; no temporal adjacency index", + "derived_files": inventory, + "seconds": time.monotonic() - start, + "ingest_config_sha256": file_digest(output / "ingest.json"), + }, + ) + budget.check() + write_json( + output / "_COMPLETE.json", + {"preparation_sha256": file_digest(output / "preparation.json")}, + ) + return config diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py new file mode 100644 index 000000000..c3d0c3c02 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py @@ -0,0 +1,241 @@ +"""Freeze cohorts, context, and future labels outside payload timing.""" + +from datetime import UTC, datetime + +import pyarrow as pa +import pyarrow.parquet as pq + +from .generate import DAY_US +from .storage import canonical_hash, connect, file_digest, quote, write_json + + +def as_time(microseconds): + return datetime.fromtimestamp(microseconds / 1000000, UTC) + + +def label_counts(con, manifest, instances): + """Count target events in [anchor, anchor+30d), including zero-event entities.""" + recipe = manifest["recipe"] + table = recipe["tables"][recipe["label_table"]] + entity = recipe["entity_table"] + fk = next(r["column"] for r in table["fkeys"] if r["table"] == entity) + con.register( + "label_instances", + instances.select(["instance_id", "entity_key", "anchor_time"]), + ) + try: + return con.sql(f""" + SELECT a.instance_id, count(f.{quote(table["primary_key"])})::BIGINT AS target + FROM label_instances a LEFT JOIN {quote(recipe["label_table"])} f + ON a.entity_key = f.{quote(fk)} + AND f.{quote(table["time_column"])} >= a.anchor_time + AND f.{quote(table["time_column"])} < a.anchor_time + INTERVAL 30 DAY + GROUP BY a.instance_id ORDER BY a.instance_id + """).to_arrow_table() + finally: + con.unregister("label_instances") + + +def build_tasks(root, manifest, budget): + cfg = manifest["recipe"]["config"] + recipe = manifest["recipe"] + pred_us = recipe["history_start_us"] + (cfg["history_days"] - 30) * DAY_US + ctx_us = pred_us - 30 * DAY_US + entity = recipe["entity_table"] + entity_key = recipe["tables"][entity]["primary_key"] + con = connect(root, manifest) + try: + con.execute("SET TimeZone='UTC'").fetchall() + parts = [] + for name, table in recipe["tables"].items(): + for rel in table["fkeys"]: + if rel["table"] == entity and table["time_column"]: + parts.append(f"""SELECT {quote(rel["column"])} AS entity_key FROM {quote(name)} + WHERE {quote(table["time_column"])} >= make_timestamp({pred_us - cfg["lookback_days"] * DAY_US}) + AND {quote(table["time_column"])} < make_timestamp({pred_us}) + AND {quote(rel["column"])} IS NOT NULL""") + con.execute(f"""CREATE TEMP TABLE degrees AS + SELECT e.{quote(entity_key)} AS entity_key, coalesce(d.degree,0)::BIGINT AS degree + FROM {quote(entity)} e LEFT JOIN + (SELECT entity_key, count(*) degree FROM ({" UNION ALL ".join(parts)}) GROUP BY entity_key) d + ON e.{quote(entity_key)} = d.entity_key""").fetchall() + median = con.execute( + "SELECT approx_quantile(degree, 0.5) FROM degrees" + ).fetchall()[0][0] + contexts = con.execute(f"""SELECT entity_key FROM degrees + ORDER BY hash(entity_key, {cfg["seed"]}), entity_key LIMIT {cfg["context_rows"]}""").fetchall() + pred_records, cohorts = [], {} + for cohort, order in [ + ("quiet", "degree ASC"), + ("typical", f"abs(degree - {median}) ASC"), + ("high_degree", "degree DESC"), + ]: + rows = con.execute(f"""SELECT entity_key, degree FROM degrees + ORDER BY {order}, hash(entity_key, {cfg["seed"]}), entity_key + LIMIT {cfg["prediction_rows"]}""").fetchall() + members = [] + for key, degree in rows: + instance_id = 1000000 + len(pred_records) + pred_records.append( + { + "instance_id": instance_id, + "entity_key": key, + "anchor_time": as_time(pred_us), + } + ) + members.append( + {"instance_id": instance_id, "entity_key": key, "degree": degree} + ) + cohorts[cohort] = members + key_type = pa.string() if cfg["preset"] == "marketo_v1" else pa.int64() + base_schema = pa.schema( + [ + pa.field("instance_id", pa.int64(), False), + pa.field("entity_key", key_type, False), + pa.field("anchor_time", pa.timestamp("us", tz="UTC"), False), + ] + ) + context = pa.Table.from_pylist( + [ + {"instance_id": i, "entity_key": row[0], "anchor_time": as_time(ctx_us)} + for i, row in enumerate(contexts) + ], + schema=base_schema, + ) + predict = pa.Table.from_pylist(pred_records, schema=base_schema) + ctx_counts = label_counts(con, manifest, context)["target"].to_pylist() + pred_counts = label_counts(con, manifest, predict)["target"].to_pylist() + degree_stats = con.execute("""SELECT min(degree), approx_quantile(degree, [0.5,0.9,0.99]), + max(degree), count(*) FILTER (WHERE degree > 0) FROM degrees""").fetchall()[0] + manifest["cohorts"] = cohorts + manifest["degree_stats"] = dict( + zip(["min", "quantiles_50_90_99", "max", "active_entities"], degree_stats) + ) + manifest["selection"] = { + "algorithm": "degree-order-with-duckdb-hash-tiebreak-v1", + "seed": cfg["seed"], + "prediction_cutoff": as_time(pred_us).isoformat(), + "context_cutoff": as_time(ctx_us).isoformat(), + } + if cfg["preset"] == "commerce_v1": + con.register("selected_users", predict.select(["entity_key"])) + overlap = con.execute(f"""SELECT count(*), coalesce(max(users),0), + count(*) FILTER (WHERE users > 1) FROM ( + SELECT item_id, count(DISTINCT user_id) users FROM interactions + WHERE user_id IN (SELECT entity_key FROM selected_users) + AND item_id IS NOT NULL AND event_time < make_timestamp({pred_us}) + AND event_time >= make_timestamp({pred_us - cfg["lookback_days"] * DAY_US}) GROUP BY item_id)""").fetchall()[0] + manifest["selected_item_overlap"] = dict( + zip(["items", "max_users_per_item", "items_shared_by_users"], overlap) + ) + for task in ("event_binary", "event_count"): + folder = root / "tasks" / task + folder.mkdir(parents=True) + targets = ( + (lambda values: [int(v > 0) for v in values]) + if task == "event_binary" + else list + ) + ctx = context.append_column( + pa.field("target", pa.int64(), False), + pa.array(targets(ctx_counts), pa.int64()), + ) + ctx = ctx.append_column( + pa.field("label_available_time", pa.timestamp("us", tz="UTC"), False), + pa.array([as_time(pred_us)] * len(ctx), pa.timestamp("us", tz="UTC")), + ) + labels = pa.table( + { + "instance_id": predict["instance_id"], + "target": pa.array(targets(pred_counts), pa.int64()), + } + ) + artifacts = {} + for name, data in [ + ("context", ctx), + ("predict", predict), + ("labels", labels), + ]: + path = folder / f"{name}.parquet" + pq.write_table(data, path, compression="zstd") + artifacts[name] = { + "path": path.relative_to(root).as_posix(), + "sha256": file_digest(path), + "rows": len(data), + } + manifest["tasks"][task] = { + "type": "classification" if task == "event_binary" else "regression", + "entity_table": entity, + "label_table": recipe["label_table"], + "label_interval": "[anchor, anchor + 30 days)", + "artifacts": artifacts, + } + for cohort, members in cohorts.items(): + for size in sorted( + {min(n, len(members)) for n in (1, 32, cfg["prediction_rows"])} + ): + paths = ( + [ + ["leads", "engagements"], + ["leads", "metrics"], + ["leads", "usecase_metrics"], + ["leads", "conversions"], + ["leads", "activities"], + ["leads", "notes"], + ] + if cfg["preset"] == "marketo_v1" + else [ + ["users", "interactions", "items"], + ["users", "interactions", "items", "categories"], + ] + ) + workload = { + "task": task, + "logical_id": manifest["logical_id"], + "snapshot_id": manifest["snapshot_id"], + "cohort": cohort, + "context_instance_ids": context["instance_id"].to_pylist(), + "prediction_instance_ids": [ + r["instance_id"] for r in members[:size] + ], + "paths": paths, + "event_fanout": 16, + "forward_fk_fanout": 1, + "lookback_days": cfg["lookback_days"], + "history_interval": "[anchor-lookback, anchor)", + "event_order": ["timestamp DESC", "primary_key ASC"], + "timeless_dimensions": [ + n + for n, t in recipe["tables"].items() + if not t["time_column"] + ], + "identity_policy": "deduplicate source IDs within each instance; preserve edges and per-instance occurrences", + "projection": { + n: [c["name"] for c in t["columns"]] + for n, t in recipe["tables"].items() + }, + "feature_columns": { + n: [ + c["name"] + for c in t["columns"] + if c["role"] != "primary_key" + and c["name"] not in {r["column"] for r in t["fkeys"]} + ] + for n, t in recipe["tables"].items() + }, + } + workload["workload_id"] = canonical_hash(workload) + path = root / "workloads" / f"{task}-{cohort}-{size}.json" + path.parent.mkdir(exist_ok=True) + write_json(path, workload) + manifest["workloads"] = [ + {"path": p.relative_to(root).as_posix(), "sha256": file_digest(p)} + for p in sorted((root / "workloads").glob("*.json")) + ] + import duckdb + + manifest["versions"]["duckdb"] = duckdb.__version__ + budget.check() + finally: + con.close() + # DuckDB deletes its own temporary files when closed; retain the directory. diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py new file mode 100644 index 000000000..e042be357 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py @@ -0,0 +1,282 @@ +"""Streaming content checks plus spillable SQL integrity/label validation.""" + +import json +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from .config import Config +from .generate import Budget, schema_for +from .storage import ( + ColumnDigests, + canonical_hash, + connect, + file_digest, + quote, + safe_path, +) +from .tasks import label_counts + + +def validate(root: Path, *, allow_incomplete=False): + root = root.resolve() + manifest = json.loads((root / "manifest.json").read_text()) + if manifest.get("format") != "fm-dataset-v1": + raise ValueError("Unsupported bundle format") + if not allow_incomplete: + marker = json.loads((root / "_COMPLETE.json").read_text()) + for name, key in [ + ("manifest.json", "manifest_sha256"), + ("validation.json", "validation_sha256"), + ]: + if file_digest(root / name) != marker[key]: + raise ValueError(f"Completion checksum mismatch: {name}") + cfg = Config(**manifest["recipe"]["config"]) + imported = manifest["recipe"].get("source_kind") == "relbench" + if imported: + from .relbench import imported_schema, validate_recipe, validate_tasks + + cfg.resolve() + recipe = validate_recipe(root, manifest) + else: + recipe = cfg.resolve() + if recipe != manifest["recipe"] or set(manifest["tables"]) != set(recipe["tables"]): + raise ValueError("Recipe/schema mismatch") + budget = Budget(root, cfg) + report = { + "status": "pass", + "tables": {}, + "tasks": {}, + "logical_id": manifest["logical_id"], + } + for name, info in manifest["tables"].items(): + table = recipe["tables"][name] + schema = imported_schema(table) if imported else schema_for(table, cfg) + if info["schema"] != schema.to_string(): + raise ValueError(f"Manifest schema mismatch: {name}") + digests, count, previous = ColumnDigests(schema), 0, None + nulls = {f.name: 0 for f in schema} + if not info["files"]: + raise ValueError(f"Missing source files: {name}") + for item in info["files"]: + path = safe_path(root, item["path"]) + if ( + file_digest(path) != item["sha256"] + or path.stat().st_size != item["bytes"] + ): + raise ValueError(f"File checksum/size mismatch: {path}") + pf = pq.ParquetFile(path) + groups = [ + pf.metadata.row_group(i).num_rows + for i in range(pf.metadata.num_row_groups) + ] + if groups != item["row_groups"] or any( + n > cfg.row_group_rows for n in groups + ): + raise ValueError(f"Row-group mismatch: {name}") + if not pf.schema_arrow.equals(schema): + raise ValueError(f"Schema mismatch: {name}") + for batch in pf.iter_batches(batch_size=65536): + count += len(batch) + digests.update(batch) + for field, array in zip(schema, batch.columns, strict=True): + nulls[field.name] += array.null_count + if not field.nullable and array.null_count: + raise ValueError(f"Null required value: {name}.{field.name}") + keys = batch.column(schema.get_field_index(table["primary_key"])) + if len(keys) and not imported: + if previous is not None and keys[0].as_py() <= previous: + raise ValueError(f"Primary keys not strictly ordered: {name}") + if ( + len(keys) > 1 + and pc.any( + pc.less_equal(keys.slice(1), keys.slice(0, len(keys) - 1)) + ).as_py() + ): + raise ValueError(f"Duplicate/unordered primary keys: {name}") + previous = keys[-1].as_py() + if ( + count != table["rows"] + or count != info["rows"] + or digests.hexdigest() != info["logical_sha256"] + ): + raise ValueError(f"Row count/logical checksum mismatch: {name}") + report["tables"][name] = { + "rows": count, + "null_counts": nulls, + "relationships": {}, + } + budget.check() + logical = canonical_hash( + { + n: {k: t[k] for k in ("rows", "schema", "logical_sha256")} + for n, t in manifest["tables"].items() + } + ) + if ( + logical != manifest["logical_id"] + or canonical_hash(manifest["tables"]) != manifest["snapshot_id"] + ): + raise ValueError("Dataset identity mismatch") + con = connect(root, manifest) + try: + for name, table in recipe["tables"].items(): + if imported: + pk = quote(table["primary_key"]) + total, unique = con.execute( + f"SELECT count(*), count(DISTINCT {pk}) FROM {quote(name)}" + ).fetchall()[0] + if total != unique: + raise ValueError(f"Duplicate/null primary keys: {name}") + if ( + table["key_policy"] == "zero-based-pinned-source-row-offset" + and total + ): + low, high = con.execute( + f"SELECT min({pk}), max({pk}) FROM {quote(name)}" + ).fetchall()[0] + if (low, high) != (0, total - 1): + raise ValueError("Invalid source occurrence IDs") + for rel in table["fkeys"]: + fk = quote(rel["column"]) + missing = con.execute(f"""SELECT count(*) FROM {quote(name)} f + ANTI JOIN {quote(rel["table"])} p ON f.{fk}=p.{quote(rel["key"])} + WHERE f.{fk} IS NOT NULL""").fetchall()[0][0] + linked, active = con.execute( + f"SELECT count({fk}), count(DISTINCT {fk}) FROM {quote(name)}" + ).fetchall()[0] + if ( + missing + or (not imported and linked != rel["linked_rows"]) + or (not imported and active != rel["active_parents"]) + ): + raise ValueError( + f"Foreign-key integrity/coverage mismatch: {name}.{rel['column']}" + ) + degree = con.execute(f"""SELECT min(n), approx_quantile(n, [0.5,0.9,0.99]), max(n) + FROM (SELECT count(*) n FROM {quote(name)} WHERE {fk} IS NOT NULL GROUP BY {fk})""").fetchall()[0] + report["tables"][name]["relationships"][rel["column"]] = { + "linked_rows": linked, + "active_parents": active, + "missing": missing, + "nonzero_degree_min": degree[0], + "nonzero_degree_quantiles_50_90_99": degree[1], + "nonzero_degree_max": degree[2], + } + if table["time_column"]: + bounds = con.execute( + f"SELECT min({quote(table['time_column'])})::VARCHAR, max({quote(table['time_column'])})::VARCHAR FROM {quote(name)}" + ).fetchall()[0] + report["tables"][name]["time_range"] = list(bounds) + if imported: + instance_sets = validate_tasks(root, manifest, con, report) + else: + instance_sets = {} + if set(manifest["tasks"]) != {"event_binary", "event_count"}: + raise ValueError("Expected both generated tasks") + for task, spec in manifest["tasks"].items(): + data = {} + for kind, item in spec["artifacts"].items(): + path = safe_path(root, item["path"]) + if file_digest(path) != item["sha256"]: + raise ValueError(f"Task checksum mismatch: {task}/{kind}") + data[kind] = pq.read_table(path) + if len(data[kind]) != item["rows"]: + raise ValueError("Task row count mismatch") + ids = data[kind]["instance_id"].to_pylist() + if len(set(ids)) != len(ids): + raise ValueError("Duplicate task instance IDs") + if "target" in data["predict"].column_names: + raise ValueError("Prediction target leakage") + for kind in ("context", "predict"): + instances = data[kind] + if not len(instances) or any( + instances[c].null_count for c in instances.column_names + ): + raise ValueError("Empty task or null instance fields") + con.register("validate_instances", instances) + entity = recipe["entity_table"] + entity_key = recipe["tables"][entity]["primary_key"] + missing = con.execute( + f"SELECT count(*) FROM validate_instances a ANTI JOIN {quote(entity)} e ON a.entity_key=e.{quote(entity_key)}" + ).fetchall()[0][0] + con.unregister("validate_instances") + if missing: + raise ValueError("Task references a missing entity") + history_end = datetime.fromtimestamp( + recipe["history_start_us"] / 1000000, UTC + ) + timedelta(days=cfg.history_days) + if any( + t + timedelta(days=30) > history_end + for t in instances["anchor_time"].to_pylist() + ): + raise ValueError("Incomplete label observation interval") + actual = label_counts(con, manifest, data[kind])[ + "target" + ].to_pylist() + expected = data["context" if kind == "context" else "labels"][ + "target" + ].to_pylist() + if spec["type"] == "classification": + actual = [int(v > 0) for v in actual] + if actual != expected: + raise ValueError(f"Label mismatch: {task}/{kind}") + if ( + data["predict"]["instance_id"].to_pylist() + != data["labels"]["instance_id"].to_pylist() + ): + raise ValueError("Prediction/label alignment mismatch") + if ( + pc.max(data["context"]["label_available_time"]).as_py() + > pc.min(data["predict"]["anchor_time"]).as_py() + ): + raise ValueError("Context labels not available at prediction time") + if any( + available != anchor + timedelta(days=30) + for anchor, available in zip( + data["context"]["anchor_time"].to_pylist(), + data["context"]["label_available_time"].to_pylist(), + strict=True, + ) + ): + raise ValueError("Incorrect context label availability") + instance_sets[task] = { + kind: set(data[kind]["instance_id"].to_pylist()) + for kind in ("context", "predict") + } + report["tasks"][task] = {k: len(v) for k, v in data.items()} + if not manifest["tasks"] or not manifest["workloads"]: + raise ValueError("Missing tasks/workloads") + for item in manifest["workloads"]: + path = safe_path(root, item["path"]) + if file_digest(path) != item["sha256"]: + raise ValueError("Workload checksum mismatch") + work = json.loads(path.read_text()) + identifier = work.pop("workload_id") + if ( + canonical_hash(work) != identifier + or work["snapshot_id"] != manifest["snapshot_id"] + or work["logical_id"] != manifest["logical_id"] + ): + raise ValueError("Workload identity mismatch") + if work["task"] not in instance_sets: + raise ValueError("Unknown workload task") + for split in ("context", "predict"): + key = ( + "context_instance_ids" + if split == "context" + else "prediction_instance_ids" + ) + ids = work[key] + if ( + not ids + or len(set(ids)) != len(ids) + or not set(ids) <= instance_sets[work["task"]][split] + ): + raise ValueError("Workload references missing/duplicate instances") + budget.check() + finally: + con.close() + return report diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py new file mode 100644 index 000000000..b9dbcd298 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py @@ -0,0 +1,66 @@ +"""HTTP encodings over retained Arrow tables.""" +import base64 + +import pyarrow as pa +import pyarrow.compute as pc + +from .bundle import TYPES +from ._phase import phase +from . import native_json + + +def encode_arrow_table(table, columns, encoding, budget, *, native_arrays=False): + """One final scalar conversion for arrays; Arrow IPC retains typed columns.""" + if len(columns) > budget["max_columns"]: + raise ValueError("Payload table exceeds column budget") + with phase("normalize_wire_values", "type_conversion"): + schema = pa.schema([pa.field(c, TYPES[s["dtype"]], nullable=s["nullable"]) + for c, s in columns.items()]) + table = table.select(list(columns)).cast(schema) + for field in schema: + if not field.nullable and table[field.name].null_count: + raise ValueError(f"Null in required Arrow field: {field.name}") + if encoding == "arrays": + if native_arrays: + # Formatting and serialization are fused; charge the combined work + # to JSON rather than pretending to split a single native call. + with phase("native_arrays_encode", "json"): + rows = native_json.encode_rows(table, budget["max_payload_bytes"]) + return {"format": "arrays", "columns": list(columns), "rows": rows}, 0 + with phase("arrays_conversion", "type_conversion"): + values = [] + for field, column in zip(schema, table.columns, strict=True): + if pa.types.is_timestamp(field.type): + # Arrow's %S retains the timestamp unit's fractional digits. + with phase("arrays_temporal_format", "type_conversion", column=field.name): + column = pc.binary_join_element_wise( + pc.strftime(column, format="%Y-%m-%dT%H:%M:%S"), "Z", "") + elif pa.types.is_date(field.type): + with phase("arrays_temporal_format", "type_conversion", column=field.name): + column = pc.strftime(column, format="%Y-%m-%d") + with phase("arrays_to_python", "type_conversion", column=field.name): + cells = column.to_pylist() + if pa.types.is_int64(field.type): + with phase("arrays_integer_bounds", "type_conversion", column=field.name): + bounds = pc.min_max(column).as_py() + if ((bounds["min"] is not None and bounds["min"] < -(2**53 - 1)) + or (bounds["max"] is not None and bounds["max"] > 2**53 - 1)): + with phase("arrays_large_integer_format", "type_conversion", column=field.name): + cells = [str(v) if v is not None and abs(v) > 2**53 - 1 else v for v in cells] + values.append(cells) + with phase("arrays_row_transpose", "type_conversion"): + return {"format": "arrays", "columns": list(columns), + "rows": list(zip(*values, strict=True))}, 0 + return (table, table.nbytes) if encoding == "arrow" else encode_ipc_table(table) + + +def encode_ipc_table(table): + """Shared IPC/base64 boundary for row and columnar inputs.""" + with phase("arrow_ipc_encode", "arrow_ipc"): + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, table.schema, options=pa.ipc.IpcWriteOptions(compression=None)) as writer: + writer.write_table(table) + raw = sink.getvalue().to_pybytes() + with phase("base64_encode", "base64"): + encoded = base64.b64encode(raw).decode() + return {"format": "arrow_ipc", "transport": "base64", "payload_base64": encoded}, len(raw) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp b/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp new file mode 100644 index 000000000..5e5faf932 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_native_json.cpp @@ -0,0 +1,238 @@ +#define PY_SSIZE_T_CLEAN +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +// The GIL stays held: PyOS_double_to_string provides exactly CPython's JSON +// float spelling without constructing a Python float for each Arrow value. +struct PythonError {}; +struct Writer { + std::string data; + size_t limit; + explicit Writer(size_t limit) : limit(limit) {} + void append(const char* s, size_t n) { + if (n > limit - data.size()) + throw std::invalid_argument("Payload exceeds byte budget; reduce the workload batch"); + data.append(s, n); + } + void append(const char* s) { append(s, std::strlen(s)); } + void put(char c) { append(&c, 1); } + void integer(int64_t n) { + char buf[32]; + auto end = std::to_chars(buf, buf + sizeof(buf), n).ptr; + append(buf, static_cast(end - buf)); + } + void hex(uint32_t cp) { + static constexpr char digits[] = "0123456789abcdef"; + char buf[] = {'\\', 'u', digits[(cp >> 12) & 15], digits[(cp >> 8) & 15], + digits[(cp >> 4) & 15], digits[cp & 15]}; + append(buf, sizeof(buf)); + } + // Match json.dumps(ensure_ascii=True), including supplementary pairs. + void string(const char* s, int64_t n) { + put('"'); + int64_t i = 0; + while (i < n) { + int64_t begin = i; + while (i < n && static_cast(s[i]) >= 0x20 && + static_cast(s[i]) < 0x7f && s[i] != '"' && s[i] != '\\') ++i; + append(s + begin, static_cast(i - begin)); + if (i == n) break; + uint32_t cp = static_cast(s[i++]); + switch (cp) { + case '"': append("\\\""); continue; + case '\\': append("\\\\"); continue; + case '\b': append("\\b"); continue; + case '\f': append("\\f"); continue; + case '\n': append("\\n"); continue; + case '\r': append("\\r"); continue; + case '\t': append("\\t"); continue; + } + if (cp < 0x80) { hex(cp); continue; } + int continuation; + uint32_t minimum; + if (cp >= 0xc2 && cp <= 0xdf) { continuation = 1; minimum = 0x80; cp &= 31; } + else if (cp >= 0xe0 && cp <= 0xef) { continuation = 2; minimum = 0x800; cp &= 15; } + else if (cp >= 0xf0 && cp <= 0xf4) { continuation = 3; minimum = 0x10000; cp &= 7; } + else throw std::invalid_argument("Invalid UTF-8 in Arrow string"); + if (continuation > n - i) throw std::invalid_argument("Truncated UTF-8 in Arrow string"); + for (int j = 0; j < continuation; ++j) { + uint8_t byte = static_cast(s[i++]); + if ((byte & 0xc0) != 0x80) throw std::invalid_argument("Invalid UTF-8 continuation"); + cp = (cp << 6) | (byte & 63); + } + if (cp < minimum || cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff)) + throw std::invalid_argument("Invalid Unicode code point in Arrow string"); + if (cp <= 0xffff) hex(cp); + else { cp -= 0x10000; hex(0xd800 + (cp >> 10)); hex(0xdc00 + (cp & 1023)); } + } + put('"'); + } + void temporal(int64_t value, bool date) { + int64_t seconds, micros = 0; + if (date) seconds = value * 86400; + else { + seconds = value / 1000000; + micros = value % 1000000; + if (micros < 0) { micros += 1000000; --seconds; } + } + time_t raw = static_cast(seconds); + struct tm parts {}; + if (static_cast(raw) != seconds || !gmtime_r(&raw, &parts)) + throw std::invalid_argument("Arrow timestamp outside platform calendar range"); + int64_t year = static_cast(parts.tm_year) + 1900; + put('"'); + if (year < 0) { put('-'); year = -year; } + if (year < 1000) put('0'); + if (year < 100) put('0'); + if (year < 10) put('0'); + integer(year); + char buf[32]; + int n = PyOS_snprintf(buf, sizeof(buf), "-%02d-%02d", parts.tm_mon + 1, parts.tm_mday); + append(buf, n); + if (!date) { + n = PyOS_snprintf(buf, sizeof(buf), "T%02d:%02d:%02d.%06dZ", + parts.tm_hour, parts.tm_min, parts.tm_sec, static_cast(micros)); + append(buf, n); + } + put('"'); + } +}; + +enum class Type { Int64, Double, Bool, String, Date, Timestamp }; +Type type_of(const ArrowSchema* s) { + if (!s || !s->format || s->dictionary || s->n_children) + throw std::invalid_argument("Native arrays encoder requires normalized scalar Arrow columns"); + const char* f = s->format; + if (!std::strcmp(f, "l")) return Type::Int64; + if (!std::strcmp(f, "g")) return Type::Double; + if (!std::strcmp(f, "b")) return Type::Bool; + if (!std::strcmp(f, "u")) return Type::String; + if (!std::strcmp(f, "tdD")) return Type::Date; + if (!std::strcmp(f, "tsu:UTC")) return Type::Timestamp; + throw std::invalid_argument(std::string("Unsupported native arrays Arrow format: ") + f); +} + +bool bit(const void* buffer, int64_t index) { + return (static_cast(buffer)[index >> 3] >> (index & 7)) & 1; +} +void cell(Writer& out, Type type, const ArrowArray* a, int64_t row) { + int64_t i = a->offset + row; + if (a->buffers[0] && !bit(a->buffers[0], i)) { out.append("null"); return; } + switch (type) { + case Type::Int64: { + int64_t v = static_cast(a->buffers[1])[i]; + bool quote = v < -9007199254740991LL || v > 9007199254740991LL; + if (quote) out.put('"'); + out.integer(v); + if (quote) out.put('"'); + break; + } + case Type::Double: { + double v = static_cast(a->buffers[1])[i]; + if (!std::isfinite(v)) throw std::invalid_argument("Out of range float values are not JSON compliant"); + char* text = PyOS_double_to_string(v, 'r', 0, Py_DTSF_ADD_DOT_0, nullptr); + if (!text) throw PythonError{}; + try { out.append(text); } catch (...) { PyMem_Free(text); throw; } + PyMem_Free(text); + break; + } + case Type::Bool: out.append(bit(a->buffers[1], i) ? "true" : "false"); break; + case Type::String: { + const auto* offsets = static_cast(a->buffers[1]); + int64_t start = offsets[i], end = offsets[i + 1]; + if (start < 0 || end < start) throw std::invalid_argument("Invalid Arrow string offsets"); + // Empty buffers may legally have a null data pointer. + const char* bytes = end == start ? "" : static_cast(a->buffers[2]) + start; + out.string(bytes, end - start); + break; + } + case Type::Date: out.temporal(static_cast(a->buffers[1])[i], true); break; + case Type::Timestamp: out.temporal(static_cast(a->buffers[1])[i], false); break; + } +} +struct SchemaOwner { + ArrowSchema value {}; + ~SchemaOwner() { if (value.release) value.release(&value); } +}; +struct ArrayOwner { + ArrowArray value {}; + ~ArrayOwner() { if (value.release) value.release(&value); } +}; +void stream_error(ArrowArrayStream* stream) { + const char* error = stream->get_last_error(stream); + throw std::invalid_argument(error ? error : "Arrow stream failed"); +} +PyObject* encode_rows(PyObject*, PyObject* args) { + PyObject* capsule; + Py_ssize_t limit; + if (!PyArg_ParseTuple(args, "On:encode_rows", &capsule, &limit)) return nullptr; + if (limit < 0) { PyErr_SetString(PyExc_ValueError, "Negative payload byte budget"); return nullptr; } + auto* stream = static_cast(PyCapsule_GetPointer(capsule, "arrow_array_stream")); + if (!stream) return nullptr; + if (!stream->release || !stream->get_schema || !stream->get_next || !stream->get_last_error) { + PyErr_SetString(PyExc_ValueError, "Released or invalid Arrow stream"); return nullptr; + } + // Borrow the stream for this synchronous call. Its PyCapsule owns/releases + // it; each schema and batch obtained here has its own release callback. + try { + SchemaOwner schema; + if (stream->get_schema(stream, &schema.value)) stream_error(stream); + const auto& s = schema.value; + if (!s.release || !s.format || std::strcmp(s.format, "+s") || s.dictionary || s.n_children < 0) + throw std::invalid_argument("Native arrays encoder requires an Arrow table stream"); + std::vector types; + for (int64_t j = 0; j < s.n_children; ++j) types.push_back(type_of(s.children[j])); + Writer out(static_cast(limit)); + out.put('['); + bool first = true; + while (true) { + ArrayOwner batch; + if (stream->get_next(stream, &batch.value)) stream_error(stream); + const auto& a = batch.value; + if (!a.release) break; + if (a.length < 0 || a.offset < 0 || a.n_children != s.n_children || a.null_count != 0) + throw std::invalid_argument("Invalid Arrow record batch"); + for (int64_t j = 0; j < a.n_children; ++j) { + const auto* child = a.children[j]; + int needed = types[j] == Type::String ? 3 : 2; + if (!child || child->offset < 0 || child->length < a.length + a.offset || + !child->buffers || child->n_buffers != needed) + throw std::invalid_argument("Invalid Arrow column layout"); + } + // Match zip(*columns) for the degenerate zero-column table. + for (int64_t row = 0; row < a.length && !types.empty(); ++row) { + if ((row & 4095) == 0 && PyErr_CheckSignals() < 0) throw PythonError{}; + if (!first) out.put(','); + first = false; + out.put('['); + for (int64_t j = 0; j < a.n_children; ++j) { + if (j) out.put(','); + cell(out, types[j], a.children[j], row + a.offset); + } + out.put(']'); + } + } + out.put(']'); + return PyBytes_FromStringAndSize(out.data.data(), static_cast(out.data.size())); + } catch (const PythonError&) { + if (!PyErr_Occurred()) PyErr_NoMemory(); + } catch (const std::bad_alloc&) { PyErr_NoMemory(); } + catch (const std::exception& e) { PyErr_SetString(PyExc_ValueError, e.what()); } + return nullptr; +} +PyMethodDef methods[] = {{"encode_rows", encode_rows, METH_VARARGS, + "Encode normalized Arrow C stream rows as compact ASCII JSON bytes."}, + {nullptr, nullptr, 0, nullptr}}; +PyModuleDef module = {PyModuleDef_HEAD_INIT, "_native_json", nullptr, -1, methods}; +} // namespace +PyMODINIT_FUNC PyInit__native_json() { return PyModule_Create(&module); } diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_phase.py b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py new file mode 100644 index 000000000..b2e919ade --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py @@ -0,0 +1,6 @@ +"""Small phase boundary for shared assembly code.""" +from contextlib import nullcontext + + +def phase(*_args, **_kwargs): + return nullcontext() diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py new file mode 100644 index 000000000..b5319cb82 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py @@ -0,0 +1,270 @@ +"""Public kumo.pytorch columnar adapter backed by DiskGraph.""" +import hashlib +import importlib.metadata +import json +from contextlib import ExitStack +from pathlib import Path + +import duckdb +import numpy as np +import pyarrow as pa +import pyarrow.parquet as pq + +from ._assembly import GraphAssembler +from ._contract import schema_for +from ._validation import validate_contract +from .bundle import TYPES, quote +from .native_selection import inclusive_second +from ._phase import phase + + +def compatibility_issues(workload): + """Report semantic gaps without importing optional native dependencies.""" + issues = [] + if workload.selection != "native_multihop": + issues.append("kumo-pytorch currently requires native_multihop selection") + for number, layer in enumerate(workload.layers): + active = [step for step in layer if step["fanout"] != 0] + if not active: + issues.append(f"layer {number}: kumo-pytorch cannot express an empty graph layer") + for step in active: + _, dest = workload.bundle.endpoint(step) + if not workload.bundle.tables[dest].get("time_column"): + issues.append(f"{step['relation']} {step['direction']}: uniform sampling into " + f"non-temporal table {dest} is unsupported by kumo-pytorch") + return list(dict.fromkeys(issues)) + + +def contract_for(workload): + issues = compatibility_issues(workload) + if issues: + raise ValueError("Cannot preserve FM sampling with kumo-pytorch:\n" + "\n".join(issues)) + from kumo.pytorch import api + bundle = workload.bundle + schema = api.DataSchema( + tables=tuple(api.TableSpec(name=name, primary_key=table["primary_key"], + columns={c: api.DType.from_arrow(TYPES[s["dtype"]]) + for c, s in table["columns"].items()}, + time_column=table.get("time_column") or api.NON_TEMPORAL) + for name, table in bundle.tables.items() if name in workload.table_names), + relations=tuple(api.RelationSpec(name=name, left_table=rel["child"], + left_key=rel["foreign_key"], right_table=rel["parent"], + right_key=bundle.tables[rel["parent"]]["primary_key"]) + for name, rel in bundle.relations.items() if name in workload.relation_names)) + layers = [] + for layer in workload.layers: + samples = [] + for step in layer: + if step["fanout"] == 0: + continue + _, dest = bundle.endpoint(step) + total = bundle.tables[dest]["rows"] + count = total if step["fanout"] is None else min(total, step["fanout"]) + if not count: + raise ValueError("kumo-pytorch migration cannot yet preserve empty destinations") + samples.append(api.GraphSample(relation=step["relation"], + direction=step["direction"], fanout=min(count, 1) + if step["direction"] == "forward" else count, strategy="recent")) + layers.append(api.GraphLayer(samples=tuple(samples))) + sample = api.SampleSpec(entity=api.EntitySpec(table=workload.entity), + neighborhoods={"fm": api.NeighborhoodSpec(layers=tuple(layers))}) + fields = {name: api.Fields(fields=tuple(api.Encode(field=column, encoding=api.Raw()) + for column in bundle.tables[name]["columns"])) for name in workload.table_names} + batch = api.BatchSpec(inputs={"graph": api.Input(source=api.NeighborhoodSource(name="fm"), + representation=api.Graph(node_fields=fields), layout=None)}, + batching=api.Batching(max_samples=workload.budgets["max_instances"])) + return api.DataLoaderSpec(schema=schema, sample=sample, batch=batch) + + + +class KumoPytorchBackend: + """Use prepare → materialize(AnchorTable) → Task.batches, without RDL. + + Inherit only FM's retained columnar graph-to-payload assembly. No direct + SamplerEngine calls or private kumo.pytorch implementation hooks are used. + """ + + merged_edge_output = True + + def __init__(self, workload, index, *, threads=2): + self.output_format = "columnar" + contract = contract_for(workload) + from kumo.pytorch import api + validate_contract(workload) + self.bundle, self.index = workload.bundle, Path(index).resolve() + if self.index == self.bundle.root or self.bundle.root in self.index.parents: + raise ValueError("Native index must be outside the immutable bundle") + self.node_types = list(self.bundle.tables) + self.assembler = GraphAssembler(self.bundle) + self.metrics = {"kumo_pytorch_batches": 0, "multihop_selected_nodes": 0, + "multihop_selected_edges": 0, "native_candidate_rows": 0, + "api": "kumo.pytorch.Task.batches", "output_format": "inspectable"} + self.derived_source_bytes = 0 + self._stack, self._tasks = ExitStack(), {} + self._workload = workload + with duckdb.connect(config={"threads": threads}) as con: + for name in workload.table_names: + table = self.bundle.tables[name] + con.read_parquet(self.bundle.paths(name)).create_view("source") + for column, spec in table["columns"].items(): + if spec["dtype"] == "timestamp[us]": + fractional = con.execute(f"SELECT count(*) FROM source WHERE " + f"epoch_us({quote(column)}) % 1000000 != 0").fetchall()[0][0] + if fractional: + raise ValueError(f"kumo-pytorch requires whole-second timestamp features: {name}.{column}") + if table.get("time_column"): + nulls = con.execute(f"SELECT count(*) FROM source WHERE " + f"{quote(table['time_column'])} IS NULL").fetchall()[0][0] + if nulls: + raise ValueError(f"kumo-pytorch would drop NULL-clock source rows: {name}") + marker = self.index / "fm_snapshot.json" + if self.index.exists() and any(self.index.iterdir()) and not marker.exists(): + raise FileExistsError("Refusing an incomplete or unrelated kumo-pytorch index") + self.index.mkdir(parents=True, exist_ok=True) + provenance = {"snapshot_id": self.bundle.snapshot_id, "adapter_version": 1, + "contract": contract.fingerprint} + provenance["dependencies"] = { + name: {"version": (dist := importlib.metadata.distribution(name)).version, + "source": json.loads(dist.read_text("direct_url.json") or "null")} + for name in ("diskgraph", "kumo-pytorch")} + if marker.exists() and json.loads(marker.read_text()) != provenance: + raise ValueError("kumo-pytorch cache belongs to another snapshot/contract") + self.preparation_mode = "cached_index_open" if marker.exists() else "index_build" + try: + # Exact file inventories: do not glob sibling files into immutable data. + source_root = self.index / "sources" + sources = {} + for name in workload.table_names: + directory = source_root / name + directory.mkdir(parents=True, exist_ok=True) + for i, path in enumerate(self.bundle.paths(name)): + link = directory / f"{i:08d}.parquet" + if not link.exists(): + link.symlink_to(path) + sources[name] = api.Parquet(path=str(directory)) + with phase("kumo_pytorch_prepare", "preparation"): + self.plan = self._stack.enter_context(api.prepare( + contract=contract, dataset=api.Dataset(tables=sources), + engine=api.DiskGraphEngine(index=self.index / "engine", num_threads=threads))) + metadata = json.loads((self.index / "engine" / "index_metadata.json").read_text()) + counts = {t["name"]: t["max_id"] + 1 for t in metadata["node_types"]} + for name in workload.table_names: + if counts.get(name, 0) != self.bundle.tables[name]["rows"]: + raise ValueError(f"kumo-pytorch ingestion lost source rows: {name}") + marker.write_text(json.dumps(provenance, indent=2) + "\n") + except BaseException: + self.close() + raise + + @staticmethod + def _key(seeds): + return tuple((s["entity_key"], s["anchor_time"]) for s in seeds) + + def prepare_batch(self, seeds): + """Materialize this exact ordered request outside request timing.""" + from kumo.pytorch import api + key = self._key(seeds) + if key in self._tasks: + return + token = hashlib.sha256(repr(key).encode()).hexdigest() + path = self.index / f"anchors-{token}.parquet" + pk = self.bundle.tables[self._workload.entity]["primary_key"] + dtype = TYPES[self.bundle.tables[self._workload.entity]["columns"][pk]["dtype"]] + # Graph neighborhoods use the native inclusive whole-second cutoff. + anchors = pa.table({"entity": pa.array([s["entity_key"] for s in seeds], dtype), + "sample": pa.array(range(len(seeds)), pa.int64()), + "at": pa.array([inclusive_second(s["anchor_time"]) for s in seeds], + pa.timestamp("s", tz="UTC"))}) + pq.write_table(anchors, path) + with phase("kumo_pytorch_materialize", "preparation"): + task = self._stack.enter_context(self.plan.materialize(api.AnchorTable( + source=api.Parquet(path=str(path)), entity_id="entity", sample_id="sample", + as_of_time="at"))) + if task.count() != len(seeds): + raise ValueError("kumo-pytorch materialization lost anchors") + self._tasks[key] = task + + def extract(self, workload, seeds): + if not seeds: + return {name: pa.table({column: pa.array([], type=TYPES[spec['dtype']]) + for column, spec in table['columns'].items()}) + for name, table in schema_for(workload)['related_tables'].items()} + if workload.id != self._workload.id: + raise ValueError("kumo-pytorch backend is bound to its prepared workload") + key = self._key(seeds) + if key not in self._tasks: + raise ValueError("Call prepare_batch(seeds) before measuring a new request") + if (len(seeds) > workload.budgets["max_instances"] or + len(seeds) > workload.budgets["max_rows_per_table"]): + raise ValueError("Native-multihop root budget exceeded") + with phase("kumo_pytorch_batches", "retrieval_selection"): + batches = iter(self._tasks[key].batches(batch_size=len(seeds), shuffle=False, + output_format=self.output_format)) + batch = next(batches) + if next(batches, None) is not None: + raise ValueError("Expected exactly one native anchor batch") + self.metrics["kumo_pytorch_batches"] += 1 + with phase("kumo_pytorch_graph_adaptation", "graph_assembly"): + return self._adapt_columnar_batch(workload, seeds, batch) + + + def _adapt_columnar_batch(self, workload, seeds, batch): + """Consume public Arrow columns; only reorder when anchor order requires it.""" + from kumo.pytorch.api import ColumnarGraph + graph = batch.inputs["graph"] + if not isinstance(graph, ColumnarGraph) or graph.root_type != workload.entity: + raise ValueError("Expected a columnar graph rooted at the workload entity") + permutation = np.asarray(batch.metadata["sample_ids"], dtype=np.int64) + if not np.array_equal(np.sort(permutation), np.arange(len(seeds))): + raise ValueError("kumo-pytorch anchor IDs are not a permutation of the request") + same_order = np.array_equal(permutation, np.arange(len(seeds))) + nodes, features, edges, directions, inverse = {}, {}, {}, {}, {} + for name, node in graph.nodes.items(): + type_id = self.node_types.index(name) + positions = node.owners.to_numpy(zero_copy_only=False) + if (node.owners.null_count or np.any(positions >= len(seeds)) or + np.any(positions < 0)): + raise ValueError("kumo-pytorch node owner is out of bounds") + table = node.features + if len(table) != len(node.ids) or len(table) != len(positions): + raise ValueError("kumo-pytorch columnar node/feature alignment mismatch") + owners = node.owners.cast(pa.uint64()) if same_order else pa.array(permutation[positions], pa.uint64()) + order = None + if not same_order and name == workload.entity: + order = np.arange(len(positions)) + order[:len(seeds)] = np.argsort(permutation[positions[:len(seeds)]]) + undo = np.empty(len(order), dtype=np.int64) + undo[order] = np.arange(len(order)) + inverse[name] = undo + owners, table = owners.take(pa.array(order)), table.take(pa.array(order)) + nodes[type_id] = [pa.record_batch( + [owners, pa.array(np.arange(len(positions)), pa.uint64())], names=["id_a", "id_b"])] + features[type_id] = table.to_batches() or [pa.RecordBatch.from_arrays( + [table[c].combine_chunks() for c in table.column_names], schema=table.schema)] + for rel_id, (name, edge) in enumerate(graph.edges.items()): + rel = self.bundle.relations[name] + if (edge.source_type, edge.destination_type) != (rel["child"], rel["parent"]): + raise ValueError("kumo-pytorch relation orientation differs from the schema") + columns = [] + for column, table_name in (("source", rel["child"]), ("destination", rel["parent"])): + values = edge.index[column] + if table_name in inverse: + indices = values.to_numpy(zero_copy_only=False) + if values.null_count or np.any(indices < 0) or np.any(indices >= len(inverse[table_name])): + raise ValueError("kumo-pytorch edge endpoint is out of bounds") + values = pa.chunked_array([pa.array(inverse[table_name][indices], pa.uint64())]) + columns.append(values.cast(pa.uint64())) + index = pa.Table.from_arrays(columns, names=["id_a", "id_b"]) + directions[rel_id] = name, 'forward', rel["child"], rel["parent"] + edges[rel_id] = index.to_batches() or [pa.RecordBatch.from_arrays( + [index[c].combine_chunks() for c in index.column_names], schema=index.schema)] + count = sum(len(b) for group in (*nodes.values(), *edges.values()) for b in group) + if count > workload.budgets["max_candidate_rows"]: + raise ValueError("Native-multihop node-plus-edge budget exceeded") + self.metrics["native_candidate_rows"] += count + return self.assembler._adapt(workload, seeds, np.arange(len(seeds), dtype=np.uint64), + nodes, features, edges, directions) + + + def close(self): + self._stack.close() diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_validation.py b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py new file mode 100644 index 000000000..59c3f86d7 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py @@ -0,0 +1,42 @@ +"""Preparation-time native sampling compatibility checks.""" +import duckdb + +from .bundle import quote +from ._phase import phase + + + +def validate_contract(workload): + """Preparation-time eligibility gate shared by all engines; no hidden fallback. + + Null sort keys are admitted by the upstream sampler, and subsecond clocks + cannot be indexed exactly. Reject those destinations before any measurement. + Fractional non-clock features remain supported by the existing ingest bridge. + """ + destinations = {workload.bundle.endpoint(s)[1] for layer in workload.layers for s in layer + if s['fanout'] != 0} + bundle = workload.bundle + with phase('multihop_contract_validation', 'preparation'): + with duckdb.connect(config={'threads': 2, 'memory_limit': '1GB'}) as con: + for name in sorted(destinations): + table = bundle.tables[name] + tc = table.get('time_column') + if not tc or not table['rows']: + continue + con.read_parquet(bundle.paths(name)).create_view('clock_source') + nulls, fractional, low, high = con.execute( + f'SELECT count(*) - count({quote(tc)}), ' + f'count(*) FILTER (WHERE epoch_us({quote(tc)}) % 1000000 != 0), ' + f'min(epoch_us({quote(tc)})), max(epoch_us({quote(tc)})) FROM clock_source' + ).fetchall()[0] + if nulls or fractional or (high - low) // 1000000 > 2 * (2**31 - 1): + raise ValueError(f'native_multihop requires non-null, whole-second, indexable clocks: {name}.{tc}; ' + 'use temporal_query or local_native for exact fallbacks') + for layer in workload.layers: + for step in layer: + total = bundle.tables[bundle.endpoint(step)[1]]['rows'] + fanout = total if step['fanout'] is None else min(total, step['fanout']) + if step['direction'] == 'forward': + fanout = min(fanout, 1) + if fanout > 2**32 - 1: + raise ValueError('native_multihop fanout exceeds uint32; use a bounded fanout') diff --git a/packages/experimental/kumo-fm/src/kumo/fm/batch.py b/packages/experimental/kumo-fm/src/kumo/fm/batch.py new file mode 100644 index 000000000..27947918d --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/batch.py @@ -0,0 +1,110 @@ +"""Retained FM Arrow tables and explicit HTTP encoding.""" +from __future__ import annotations + +from copy import deepcopy +from types import MappingProxyType + +import pyarrow as pa + +from ._encoding import encode_arrow_table +from ._contract import metadata_for +from .bundle import TYPES +from . import native_json + + +def _typed_table(table, columns, max_columns): + if len(columns) > max_columns: + raise ValueError("Payload table exceeds column budget") + schema = pa.schema([pa.field(name, TYPES[spec["dtype"]], nullable=spec["nullable"]) + for name, spec in columns.items()]) + table = table.select(list(columns)).cast(schema) + for field in schema: + if not field.nullable and table[field.name].null_count: + raise ValueError(f"Null in required Arrow field: {field.name}") + return table + + +class PayloadBatch: + """One sampled request; Arrow tables stay valid after loader closure.""" + + def __init__(self, workload, splits): + metadata = metadata_for(workload) + self._metadata = deepcopy(metadata) + self._budgets = dict(workload.budgets) + self._splits = {} + logical_bytes = 0 + for split, (seeds, related) in splits.items(): + columns = dict(metadata["schema"]["instance_table"]["columns"]) + if split == "predict": + columns.pop("__fm_target") + instance = pa.table({ + name: pa.array([str(seed["target"]) if name == "__fm_target" + and workload.kind != "regression" else seed[{ + "instance_id": "instance_id", + "__fm_entity_key": "entity_key", + "__fm_anchor": "anchor_time", + "__fm_target": "target", + }[name]] for seed in seeds], type=TYPES[spec["dtype"]]) + for name, spec in columns.items() + }) + instance = _typed_table(instance, columns, self._budgets["max_columns"]) + if len(instance) > self._budgets["max_rows_per_table"]: + raise ValueError("Instance table exceeds row budget") + normalized = {} + for name, spec in metadata["schema"]["related_tables"].items(): + table = _typed_table(related[name], spec["columns"], self._budgets["max_columns"]) + if len(table) > self._budgets["max_rows_per_table"]: + raise ValueError("Related table exceeds row budget") + normalized[name] = table + logical_bytes += table.nbytes + logical_bytes += instance.nbytes + self._splits[split] = (instance, MappingProxyType(normalized)) + if logical_bytes > self._budgets["max_decoded_arrow_bytes"]: + raise ValueError("Decoded Arrow budget exceeded") + self._logical_bytes = logical_bytes + + @property + def metadata(self): + return deepcopy(self._metadata) + + @property + def logical_arrow_bytes(self): + return self._logical_bytes + + @property + def splits(self): + return tuple(self._splits) + + def tables(self, split): + instance, related = self._splits[split] + return MappingProxyType({"instance_table": instance, "related_tables": related}) + + def as_arrow_request(self): + result = self.metadata + for split, (instance, related) in self._splits.items(): + result[split] = {"instance_table": instance, "related_tables": dict(related)} + return result + + def encode(self, encoding="arrays"): + if encoding not in ("arrays", "arrow_ipc"): + raise ValueError("Unknown encoding") + payload = self.metadata + for split, (instance, related) in self._splits.items(): + instance_columns = dict(payload["schema"]["instance_table"]["columns"]) + if split == "predict": + instance_columns.pop("__fm_target") + encoded_instance, _ = encode_arrow_table( + instance, instance_columns, encoding, self._budgets, native_arrays=encoding == "arrays") + encoded_related = {} + for name, table in related.items(): + encoded_related[name], _ = encode_arrow_table( + table, payload["schema"]["related_tables"][name]["columns"], encoding, + self._budgets, native_arrays=encoding == "arrays") + payload[split] = {"instance_table": encoded_instance, "related_tables": encoded_related} + if encoding == "arrays": + return native_json.dumps(payload, self._budgets["max_payload_bytes"]) + import json + body = json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() + if len(body) > self._budgets["max_payload_bytes"]: + raise ValueError("Payload exceeds byte budget; reduce the workload batch") + return body diff --git a/packages/experimental/kumo-fm/src/kumo/fm/bundle.py b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py new file mode 100644 index 000000000..c5c8db8f9 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py @@ -0,0 +1,163 @@ +"""Validated Parquet bundle metadata, shared by every backend. + +Full validation uses streaming file hashes and spillable DuckDB constraints; +only task rows and sampled graph rows need to become Python objects. +""" +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +import duckdb +import pyarrow as pa +import pyarrow.parquet as pq + +TYPES = {"int64": pa.int64(), "float64": pa.float64(), "bool": pa.bool_(), + "string": pa.string(), "date": pa.date32(), "timestamp[us]": pa.timestamp("us", tz="UTC")} + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def identity(value) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def quote(name: str) -> str: + return '"' + name.replace('"', '""') + '"' + + +def check_name(name: str) -> None: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name) or name.startswith("__") or name == "instance_id": + raise ValueError(f"Unsupported or reserved schema name: {name!r}") + + +class Bundle: + def __init__(self, root: Path, *, verify_files: bool = True): + self.root = Path(root).resolve() + self.manifest = json.loads((self.root / "manifest.json").read_text()) + m = self.manifest + self.dataset_manifest = m if m.get("format") == "fm-dataset-v1" else None + if self.dataset_manifest is None and m.get("format_version") != 2: + raise ValueError("Expected dataset-suite bundle format_version=2") + complete = json.loads((self.root / "_COMPLETE.json").read_text()) + if complete.get("manifest_sha256") != digest(self.root / "manifest.json"): + raise ValueError("Incomplete or changed bundle manifest") + validation = json.loads((self.root / "validation.json").read_text()) + if (validation.get("status") != ("pass" if self.dataset_manifest else "passed") or + complete.get("validation_sha256") != digest(self.root / "validation.json")): + raise ValueError("Bundle has no matching completed validation") + if self.dataset_manifest is not None: + from .datasets import manifest_view + self.manifest = m = manifest_view(self.dataset_manifest, self.root) + self.tables = m["tables"] + self.relations = m["relations"] + self.logical_id = m["logical_id"] + self.snapshot_id = self.dataset_manifest["snapshot_id"] if self.dataset_manifest else identity(m) + if not self.tables: + raise ValueError("Bundle must declare tables") + for name, table in self.tables.items(): + check_name(name) + columns = table["columns"] + for column, spec in columns.items(): + check_name(column) + if spec["dtype"] not in TYPES or type(spec.get("nullable")) is not bool: + raise ValueError(f"Unsupported column specification: {name}.{column}") + pk = table["primary_key"] + if pk not in columns or columns[pk]["dtype"] not in ("int64", "string") or columns[pk]["nullable"]: + raise ValueError(f"Table {name} needs a non-null int64/string primary key") + tc = table.get("time_column") + if tc is not None and (tc not in columns or columns[tc]["dtype"] not in ("timestamp[us]", "date")): + raise ValueError(f"Invalid time column: {name}") + if tc is None and table.get("availability") != "timeless": + raise ValueError(f"Declare {name} temporal or explicitly timeless") + if type(table["rows"]) is not int or table["rows"] < 0 or not table["files"]: + raise ValueError(f"Invalid table inventory: {name}") + if table["files"] != sorted(set(table["files"])): + raise ValueError(f"Table file inventory must be sorted and unique: {name}") + if any(p not in m["files"] for p in table["files"]): + raise ValueError(f"Uninventoried table file: {name}") + for name, rel in self.relations.items(): + check_name(name) + child, parent = self.tables[rel["child"]], self.tables[rel["parent"]] + fk = child["columns"][rel["foreign_key"]] + pk = parent["columns"][parent["primary_key"]] + if fk["dtype"] != pk["dtype"]: + raise ValueError(f"Incompatible relation key types: {name}") + if verify_files: + for name, entry in m["files"].items(): + if digest(self.path(name)) != entry["sha256"]: + raise ValueError(f"Bundle checksum mismatch: {name}") + + def path(self, relative: str) -> Path: + path = (self.root / relative).resolve() + if self.root not in path.parents: + raise ValueError("Bundle file path escapes the dataset directory") + return path + + def paths(self, name: str) -> list[str]: + return [str(self.path(p)) for p in self.tables[name]["files"]] + + def schema(self, name: str) -> pa.Schema: + return pa.schema([pa.field(c, TYPES[s["dtype"]], s["nullable"]) + for c, s in self.tables[name]["columns"].items()]) + + def endpoint(self, step: dict) -> tuple[str, str]: + rel = self.relations[step["relation"]] + return (rel["child"], rel["parent"]) if step["direction"] == "forward" else (rel["parent"], rel["child"]) + + def register(self, con, *, native: bool = False) -> None: + for name in self.tables: + con.read_parquet(self.paths(name)).create_view("__source_" + name) + kind = "TABLE" if native else "VIEW" + con.execute(f"CREATE {kind} {quote(name)} AS SELECT * FROM {quote('__source_' + name)}").fetchall() + if native: + con.execute("ANALYZE").fetchall() + + def validate(self, *, memory_limit: str = "1GB", threads: int = 2) -> dict: + """Full source validation; preparation cost, never timed as retrieval.""" + if getattr(self, "dataset_manifest", None) is not None: + from ._dataset_support.validate import validate + validate(self.root) + result = {"tables": {}, "relations": {}} + with duckdb.connect(config={"memory_limit": memory_limit, "threads": threads}) as con: + self.register(con) + for name, table in self.tables.items(): + expected = self.schema(name) + for path in self.paths(name): + parquet = pq.ParquetFile(path) + if not parquet.schema_arrow.equals(expected): + raise ValueError(f"Parquet schema differs from manifest: {path}") + entry = self.manifest["files"][str(Path(path).relative_to(self.root))] + if parquet.metadata.num_rows != entry["rows"]: + raise ValueError(f"File row count differs from inventory: {path}") + n = con.execute(f"SELECT count(*) FROM {quote(name)}").fetchall()[0][0] + if n != table["rows"]: + raise ValueError(f"Table row count mismatch: {name}") + for col, spec in table["columns"].items(): + if not spec["nullable"] and con.execute( + f"SELECT count(*) FROM {quote(name)} WHERE {quote(col)} IS NULL" + ).fetchall()[0][0]: + raise ValueError(f"Null required column: {name}.{col}") + if spec["dtype"] == "float64" and con.execute( + f"SELECT count(*) FROM {quote(name)} WHERE NOT isfinite({quote(col)})" + ).fetchall()[0][0]: + raise ValueError(f"Nonfinite feature: {name}.{col}") + pk = quote(table["primary_key"]) + if con.execute(f"SELECT count(*) FROM (SELECT {pk} FROM {quote(name)} GROUP BY {pk} HAVING count(*) > 1)").fetchall()[0][0]: + raise ValueError(f"Duplicate primary key: {name}") + result["tables"][name] = {"rows": n} + for name, rel in self.relations.items(): + pk = self.tables[rel["parent"]]["primary_key"] + child, parent, fk = quote(rel["child"]), quote(rel["parent"]), quote(rel["foreign_key"]) + bad = con.execute(f"SELECT count(*) FROM {child} c LEFT JOIN {parent} p ON c.{fk}=p.{quote(pk)} WHERE c.{fk} IS NOT NULL AND p.{quote(pk)} IS NULL").fetchall()[0][0] + if bad: + raise ValueError(f"Dangling foreign keys: {name} ({bad})") + nulls = con.execute(f"SELECT count(*) FROM {child} WHERE {fk} IS NULL").fetchall()[0][0] + result["relations"][name] = {"null_links": nulls, "edges": self.tables[rel["child"]]["rows"] - nulls} + self.validation = result + return result diff --git a/packages/experimental/kumo-fm/src/kumo/fm/datasets.py b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py new file mode 100644 index 000000000..60cc78ba0 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py @@ -0,0 +1,99 @@ +"""Adapt fm_datasets' published bundles without rewriting their source artifacts.""" +from copy import deepcopy + +from ._dataset_support.config import Config +from ._dataset_support.generate import DAY_US, schema_for +from ._dataset_support.storage import canonical_hash, safe_path + + +def manifest_view(source, root): + """Expose the loader's internal schema while retaining producer identities.""" + cfg = Config(**source["recipe"]["config"]) + tables, relations, files, tasks = {}, {}, {}, {} + for name, recipe in source["recipe"]["tables"].items(): + schema = schema_for(recipe, cfg) + columns = {} + for col, field in zip(recipe["columns"], schema, strict=True): + spec = {"dtype": "timestamp[us]" if col["type"] == "time" else col["type"], + "nullable": field.nullable} + if col["role"] in ("ID", "categorical", "numerical", "text", "timestamp"): + spec["stype"] = col["role"] + columns[col["name"]] = spec + inventory = source["tables"][name] + tables[name] = {"columns": columns, "primary_key": recipe["primary_key"], + "time_column": recipe["time_column"], "rows": inventory["rows"], + "files": sorted(item["path"] for item in inventory["files"])} + if not recipe["time_column"]: + tables[name]["availability"] = "timeless" + for item in inventory["files"]: + files[item["path"]] = {**item, "rows": sum(item["row_groups"])} + for rel in recipe["fkeys"]: + key = name + "_" + rel["column"] + relations[key] = {"child": name, "foreign_key": rel["column"], "parent": rel["table"]} + for name, task in source["tasks"].items(): + if task["type"] not in ("classification", "regression"): + raise ValueError("Unsupported generated task type") + target = {"dtype": "int64"} + if task["type"] == "classification": + target["classes"] = [0, 1] + tasks[name] = {"entity_table": task["entity_table"], "kind": task["type"], + "target": target, "label_horizon_us": 30 * DAY_US, + "label_available_column": "label_available_time"} + for split, item in task["artifacts"].items(): + tasks[name][split] = item["path"] + # The producer's task inventory omits size; report actual bytes. + files[item["path"]] = {**item, "bytes": safe_path(root, item["path"]).stat().st_size} + return {"format_version": 2, "logical_id": source["logical_id"], + "tables": tables, "relations": relations, "tasks": tasks, "files": files} + + +def workload_view(bundle, source): + """Resolve paths to directed layers and retain exact frozen instance order.""" + from .bundle import digest + source = deepcopy(source) + identifier = source.pop("workload_id", None) + if identifier != canonical_hash(source): + raise ValueError("Generated workload identity mismatch") + if source["logical_id"] != bundle.logical_id or source["snapshot_id"] != bundle.snapshot_id: + raise ValueError("Generated workload is bound to another snapshot") + if (source.get("history_interval") != "[anchor-lookback, anchor)" or + source.get("event_order") != ["timestamp DESC", "primary_key ASC"] or + source.get("identity_policy") != "deduplicate source IDs within each instance; preserve edges and per-instance occurrences"): + raise ValueError("Unsupported generated extraction policy") + timeless = {n for n, t in bundle.tables.items() if t.get("time_column") is None} + if set(source["timeless_dimensions"]) != timeless: + raise ValueError("Generated workload timeless dimensions differ from metadata") + layers, names = [], {bundle.manifest["tasks"][source["task"]]["entity_table"]} + for path in source["paths"]: + if len(path) < 2 or path[0] != bundle.manifest["tasks"][source["task"]]["entity_table"]: + raise ValueError("Generated paths must start at the task entity") + for depth, (src, dst) in enumerate(zip(path, path[1:])): + choices = [] + for name, rel in bundle.relations.items(): + if (rel["child"], rel["parent"]) == (src, dst): + choices.append((name, "forward")) + if (rel["parent"], rel["child"]) == (src, dst): + choices.append((name, "reverse")) + if len(choices) != 1: + raise ValueError(f"Missing or ambiguous directed path: {src} -> {dst}") + name, direction = choices[0] + step = {"relation": name, "direction": direction, + "fanout": source["forward_fk_fanout"] if direction == "forward" else source["event_fanout"]} + if len(layers) <= depth: + layers.append([]) + if step not in layers[depth]: + layers[depth].append(step) + names.update((src, dst)) + return {"version": 1, "logical_id": bundle.logical_id, "snapshot_id": bundle.snapshot_id, + "source_manifest_sha256": digest(bundle.root / "manifest.json"), "source_workload_id": identifier, + "task": source["task"], "prediction_cutoff": bundle.dataset_manifest["selection"]["prediction_cutoff"], + # Frozen cohorts/paths are reused, but the loader now deliberately + # compares local/native sampling. Retain the producer policy so the + # effective workload and report expose this semantic change. + "source_sampling_policy": {key: source[key] for key in + ("lookback_days", "history_interval", "event_order")}, + "selection": "local_native", "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", "layers": layers, + "projections": {n: source["projection"][n] for n in sorted(names)}, + "context_instance_ids": source["context_instance_ids"], + "prediction_instance_ids": source["prediction_instance_ids"], "encoding": "arrays"} diff --git a/packages/experimental/kumo-fm/src/kumo/fm/loader.py b/packages/experimental/kumo-fm/src/kumo/fm/loader.py new file mode 100644 index 000000000..42c072809 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/loader.py @@ -0,0 +1,106 @@ +"""Dataset, index, request, and batch lifecycle.""" +from __future__ import annotations + +import json +from pathlib import Path + +from ._pytorch import KumoPytorchBackend +from .batch import PayloadBatch +from .bundle import Bundle +from .workload import Workload + + +def _select(available, selected, split): + if selected is None: + return list(available) + if not isinstance(selected, (list, tuple)): + raise TypeError(f"{split} must be an ordered sequence of instance IDs") + if any(type(value) is not int for value in selected): + raise ValueError(f"{split} instance IDs must be int64 values") + by_id = {seed["instance_id"]: seed for seed in available} + if len(set(selected)) != len(selected) or not set(selected) <= by_id.keys(): + raise ValueError(f"{split} has duplicate or unknown instance IDs") + return [by_id[value] for value in selected] + + +class PreparedRequest: + """Resolved anchors; every load call samples exactly once per included split.""" + + def __init__(self, loader, splits): + self._loader = loader + self._splits = splits + self._closed = False + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + self._closed = True + + def load(self): + if self._closed or self._loader._closed: + raise RuntimeError("Prepared request or loader is closed") + workload = self._loader.workload + ready = { + split: (seeds, self._loader._backend.extract(workload, seeds)) + for split, seeds in self._splits.items() + } + return PayloadBatch(workload, ready) + + +class FMLoader: + """Open a validated bundle and its compatible reusable DiskGraph index.""" + + def __init__(self, dataset, spec, *, index, threads=2, verify_files=True): + bundle = Bundle(dataset.root if isinstance(dataset, Bundle) else Path(dataset), + verify_files=verify_files) + if isinstance(spec, Workload): + if spec.bundle.root != bundle.root: + raise ValueError("Workload is bound to another bundle") + workload = spec if spec.bundle is bundle else Workload(bundle, spec.spec) + else: + document = json.loads(Path(spec).read_text()) if isinstance(spec, (str, Path)) else spec + workload = Workload(bundle, document) + if verify_files: + bundle.validate(threads=threads) + self.workload = workload + self._backend = KumoPytorchBackend(workload, index, threads=threads) + self._closed = False + + @property + def preparation_mode(self): + return self._backend.preparation_mode + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + if not self._closed: + self._closed = True + self._backend.close() + + def materialize(self, *, context=None, predict=None): + if self._closed: + raise RuntimeError("Loader is closed") + available = self.workload.instances + splits = {} + for split, selected in (("context", context), ("predict", predict)): + seeds = _select(available[split], selected, split) + if seeds: + if len(seeds) > self.workload.budgets["max_instances"]: + raise ValueError("Instance budget exceeded") + self._backend.prepare_batch(seeds) + splits[split] = seeds + if not splits: + raise ValueError("Request needs at least one split") + return PreparedRequest(self, splits) + + +def prepare(dataset, spec, *, index, threads=2, verify_files=True): + return FMLoader(dataset, spec, index=index, threads=threads, verify_files=verify_files) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_json.py b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py new file mode 100644 index 000000000..7be2d2b53 --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py @@ -0,0 +1,68 @@ +"""Shared native arrays encoding; Python handles the small request envelope.""" +from dataclasses import dataclass +import json +import pyarrow as pa +import pyarrow.compute as pc + +from ._phase import phase + + +@dataclass(frozen=True) +class EncodedRows: + """Internal JSON fragment produced only by the native Arrow encoder.""" + data: bytes + + +def encode_rows(table, max_bytes): + # Fail explicitly if a source checkout was not built; never silently time + # the reference encoder when native encoding was requested. + from ._native_json import encode_rows as native_encode_rows + # Arrow's calendar formatter has implementation-specific behavior outside + # Python's year 1..9999 range. Preserve that existing behavior (including + # errors) with its native string kernel for unusual columns, while normal + # dates/timestamps use the direct C++ formatter. No per-cell Python values. + with phase('native_arrays_prepare', 'type_conversion'): + for index, field in enumerate(table.schema): + timestamp = pa.types.is_timestamp(field.type) + date = pa.types.is_date32(field.type) + if not (timestamp or date): + continue + column = table.column(index) + bounds = pc.min_max(column.cast(pa.int64() if timestamp else pa.int32())).as_py() + low, high = ((-62135596800000000, 253402300799999999) if timestamp else (-719162, 2932896)) + if bounds['min'] is not None and (bounds['min'] < low or bounds['max'] > high): + values = pc.strftime(column, format='%Y-%m-%dT%H:%M:%S' if timestamp else '%Y-%m-%d') + if timestamp: + values = pc.binary_join_element_wise(values, 'Z', '') + table = table.set_column(index, field.name, values) + return EncodedRows(native_encode_rows(table.__arrow_c_stream__(), max_bytes)) + + +def dumps(payload, max_bytes): + """Preserve json.dumps ordering/escaping while inserting encoded row arrays. + + Walk only the envelope dictionaries. Ordinary lists (including legacy row + lists) retain the reference JSON implementation. Native rows stay bytes. + """ + def parts(value): + if isinstance(value, EncodedRows): + yield value.data + elif isinstance(value, dict): + yield b'{' + for i, (key, item) in enumerate(value.items()): + if i: + yield b',' + yield json.dumps(key, ensure_ascii=True).encode('ascii') + yield b':' + yield from parts(item) + yield b'}' + else: + yield json.dumps(value, separators=(',', ':'), allow_nan=False).encode('ascii') + + fragments, size = [], 0 + for fragment in parts(payload): + size += len(fragment) + if size > max_bytes: + raise ValueError('Payload exceeds byte budget; reduce the workload batch') + fragments.append(fragment) + return b''.join(fragments) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py new file mode 100644 index 000000000..86ccdc17a --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py @@ -0,0 +1,8 @@ +"""Inclusive whole-second graph cutoff for native anchor tables.""" +from datetime import timedelta + +from .workload import EPOCH + + +def inclusive_second(anchor): + return (anchor - EPOCH) // timedelta(seconds=1) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/workload.py b/packages/experimental/kumo-fm/src/kumo/fm/workload.py new file mode 100644 index 000000000..2bbe7f4cc --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/workload.py @@ -0,0 +1,231 @@ +"""Frozen task instances and an explicit, bounded traversal policy.""" +from __future__ import annotations + +from datetime import date, datetime, time, timedelta, timezone +import math +from pathlib import Path +import json + +import pyarrow as pa +import pyarrow.parquet as pq + +from .bundle import TYPES, Bundle, identity + +UTC = timezone.utc +EPOCH = datetime(1970, 1, 1, tzinfo=UTC) + + +def timestamp(value) -> datetime: + if isinstance(value, date) and not isinstance(value, datetime): + value = datetime.combine(value, time(), tzinfo=UTC) + if isinstance(value, str): + value = datetime.fromisoformat(value.replace("Z", "+00:00")) + if not isinstance(value, datetime) or value.tzinfo is None: + raise ValueError("Timestamps must include a UTC offset") + return value.astimezone(UTC) + + +def validate_scalar(value, dtype: str, *, nullable: bool = False): + if value is None: + if nullable: + return + raise ValueError("Unexpected null value") + valid = ((dtype == "int64" and type(value) is int and -(2**63) <= value < 2**63) or + (dtype == "string" and isinstance(value, str)) or + (dtype == "bool" and type(value) is bool) or + (dtype == "float64" and type(value) in (int, float) and math.isfinite(value))) + if dtype == "timestamp[us]": + timestamp(value) + return + if not valid: + raise ValueError(f"Value does not match {dtype}: {value!r}") + + +class Workload: + def __init__(self, bundle: Bundle, spec: dict): + if getattr(bundle, "dataset_manifest", None) is not None and spec.get("version") != 1: + from .datasets import workload_view + spec = workload_view(bundle, spec) + self.bundle, self.spec = bundle, spec + if spec.get("version") != 1 or spec.get("logical_id") != bundle.logical_id or spec.get("snapshot_id") != bundle.snapshot_id: + raise ValueError("Workload is not bound to this logical dataset and physical snapshot") + self.task = bundle.manifest["tasks"][spec["task"]] + self.entity = self.task["entity_table"] + self.target = self.task["target"] + if self.entity not in bundle.tables or self.target["dtype"] not in ("string", "int64", "float64"): + raise ValueError("Unsupported entity/target type") + self.kind = self.task["kind"] + if self.kind not in ("binary_classification", "multiclass_classification", "classification", "regression"): + raise ValueError("Only classification and regression tasks are supported") + if self.kind == "regression": + if self.target["dtype"] not in ("float64", "int64"): + raise ValueError("Regression needs a numeric target") + else: + classes = self.target.get("classes", []) + if len(classes) < 2 or len(set(classes)) != len(classes): + raise ValueError("Classification needs distinct target classes") + if self.kind == "binary_classification" and len(classes) != 2: + raise ValueError("Binary classification needs exactly two classes") + for value in classes: + validate_scalar(value, self.target["dtype"]) + self.cutoff = timestamp(spec["prediction_cutoff"]) + if spec.get("lookback_us") is not None: + raise ValueError("lookback_us is unsupported; temporal_query uses lookback_seconds") + self.selection = spec.get("selection") + if self.selection not in ("local_native", "temporal_query", "native_multihop") or spec.get("deduplication") != "per_instance": + raise ValueError("Declare local_native, temporal_query or native_multihop selection and per-instance deduplication") + self.lookback_seconds = spec.get("lookback_seconds") + if self.lookback_seconds is not None: + if self.selection != "temporal_query": + raise ValueError("lookback_seconds requires temporal_query selection") + if type(self.lookback_seconds) is not int or not 0 < self.lookback_seconds < 2**63: + raise ValueError("lookback_seconds must be a positive int64 or null") + if spec.get("fanout_scope") != "per_source_relation_hop": + raise ValueError("fanout_scope must be per_source_relation_hop") + self.layers = spec["layers"] + frontier = {self.entity} + self.table_names, self.relation_names = {self.entity}, set() + if not isinstance(self.layers, list) or len(self.layers) > 32: + raise ValueError("Expected at most 32 explicit traversal layers") + for layer in self.layers: + next_frontier, seen = set(), set() + if not layer: + raise ValueError("Traversal layers cannot be empty") + for step in layer: + if step["direction"] not in ("forward", "reverse") or step["relation"] not in bundle.relations: + raise ValueError("Unknown directed relation") + name = (step["relation"], step["direction"]) + if name in seen: + raise ValueError("Repeated relation in one layer") + seen.add(name) + source, dest = bundle.endpoint(step) + if source not in frontier: + raise ValueError(f"Relation source is not reachable at this hop: {source}") + fanout = step["fanout"] + if fanout is not None and (type(fanout) is not int or fanout < 0): + raise ValueError("Fanout must be nonnegative or null (all)") + next_frontier.add(dest) + self.table_names.update((source, dest)) + self.relation_names.add(step["relation"]) + frontier = next_frontier + self.projections = spec["projections"] + if set(self.projections) != self.table_names: + raise ValueError("Projections must declare exactly the reachable tables, including empty tables") + for name, columns in self.projections.items(): + if len(columns) != len(set(columns)) or not set(columns) <= set(bundle.tables[name]["columns"]): + raise ValueError(f"Invalid projection for {name}") + if self.selection == "native_multihop": + from ._validation import validate_contract + validate_contract(self) + self.encoding = spec.get("encoding", "arrays") + if self.encoding not in ("arrays", "arrow_ipc"): + raise ValueError("Unknown output encoding") + self.budgets = {"max_instances": 10000, "max_rows_per_table": 1000000, + "max_payload_bytes": 30 * 1024**2, "max_decoded_arrow_bytes": 30 * 1024**2, + "max_columns": 500, "max_candidate_rows": 1000000, + **spec.get("budgets", {})} + if any(type(v) is not int or v < 1 for v in self.budgets.values()): + raise ValueError("Budgets must be positive integers") + self.instances = {split: self._instances(split) for split in ("context", "predict")} + self.id = identity(spec) + + @classmethod + def load(cls, bundle: Bundle, path: Path): + return cls(bundle, json.loads(Path(path).read_text())) + + def _instances(self, split: str) -> list[dict]: + file = self.task[split] + if file not in self.bundle.manifest["files"]: + raise ValueError("Task file is missing from the immutable inventory") + path = self.bundle.path(file) + parquet = pq.ParquetFile(path) + n = parquet.metadata.num_rows + if n != self.bundle.manifest["files"][file]["rows"]: + raise ValueError("Task row count differs from inventory") + selection = self.spec.get("context_instance_ids" if split == "context" else "prediction_instance_ids") + if selection is not None: + for value in selection: + validate_scalar(value, "int64") + if len(selection) != len(set(selection)): + raise ValueError("Duplicate selected instance IDs") + count = n if selection is None else len(selection) + if count < 1 or count > self.budgets["max_instances"]: + raise ValueError("Task instance count exceeds budget or is empty") + data = pq.read_table(path, filters=None if selection is None else [("instance_id", "in", selection)]) + available = self.task.get("label_available_column", "label_available_at") + if split == "context" and available != "label_available_at": + data = data.rename_columns(["label_available_at" if c == available else c for c in data.column_names]) + rows = data.to_pylist() + if selection is not None: + lookup = {row["instance_id"]: row for row in rows} + if len(rows) != len(lookup) or set(selection) != set(lookup): + raise ValueError("Selected task instances are missing or duplicated") + rows = [lookup[i] for i in selection] + expected = {"instance_id", "entity_key", "anchor_time"} + if split == "context": + expected |= {"target", "label_available_at"} + ids = set() + entity = self.bundle.tables[self.entity] + key_dtype = entity["columns"][entity["primary_key"]]["dtype"] + fields = {"instance_id": TYPES["int64"], "entity_key": TYPES[key_dtype], + "anchor_time": TYPES["timestamp[us]"]} + if split == "context": + fields.update(target=TYPES[self.target["dtype"]], label_available_at=TYPES["timestamp[us]"]) + schema = pa.schema([pa.field(c, t, nullable=False) for c, t in fields.items()]) + if not data.schema.equals(schema): + raise ValueError(f"Invalid {split} task schema; prediction targets must stay in labels.parquet") + horizon = self.task.get("label_horizon_us") + if horizon is not None and (type(horizon) is not int or horizon <= 0): + raise ValueError("Label horizon must be positive microseconds") + for row in rows: + if set(row) != expected: + raise ValueError(f"Unexpected {split} columns; prediction targets must stay in labels.parquet") + validate_scalar(row["instance_id"], "int64") + validate_scalar(row["entity_key"], key_dtype) + if row["instance_id"] in ids: + raise ValueError("Duplicate instance_id within a split") + ids.add(row["instance_id"]) + row["anchor_time"] = timestamp(row["anchor_time"]) + if split == "predict" and row["anchor_time"] != self.cutoff: + raise ValueError("Prediction anchors must share the workload cutoff") + if split == "context": + row["label_available_at"] = timestamp(row["label_available_at"]) + if not row["anchor_time"] <= row["label_available_at"] <= self.cutoff: + raise ValueError("Context label is unavailable at prediction cutoff") + if horizon is not None and row["label_available_at"] < row["anchor_time"] + timedelta(microseconds=horizon): + raise ValueError("Context label observation interval is incomplete") + validate_scalar(row["target"], self.target["dtype"]) + if self.kind != "regression" and row["target"] not in self.target["classes"]: + raise ValueError("Context target is outside the declared classes") + return rows + + def eligible(self, name: str, row: dict, seed: dict) -> bool: + tc = self.bundle.tables[name].get("time_column") + if tc is None: + return True + if row[tc] is None: + return False + value = timestamp(row[tc]) + anchor = seed["anchor_time"] + return value <= anchor + + def rank(self, name: str, row: dict): + table = self.bundle.tables[name] + tc = table.get("time_column") + micros = 0 if tc is None else (timestamp(row[tc]) - EPOCH) // timedelta(microseconds=1) + return -micros + + def neighbor_eligible(self, name: str, row: dict, seed: dict) -> bool: + """Window applies to destinations at every hop, never root availability.""" + if not self.eligible(name, row, seed): + return False + tc = self.bundle.tables[name].get("time_column") + if tc and self.lookback_seconds is not None: + age = (seed["anchor_time"] - timestamp(row[tc])) // timedelta(microseconds=1) + return age < self.lookback_seconds * 1000000 + return True + + def temporal_order(self, name: str, row: dict): + # query_temporal reverses both timestamp and Python-PK-repr ordering. + # Homogeneous int64/string PK types make its type-name tie key constant. + return (-self.rank(name, row), repr(row[self.bundle.tables[name]["primary_key"]])) diff --git a/packages/experimental/kumo-fm/tests/_fixture.py b/packages/experimental/kumo-fm/tests/_fixture.py new file mode 100644 index 000000000..52458eccd --- /dev/null +++ b/packages/experimental/kumo-fm/tests/_fixture.py @@ -0,0 +1,198 @@ +"""Tiny whole-second contract bundles, not a bulk generator. + +Timestamp values have one-second granularity. Parquet and FM serialization +retain timestamp[us] as the physical type, with zero subsecond components. +""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +from kumo.fm.bundle import Bundle, TYPES, digest, identity + + +def scalar(value): + if isinstance(value, datetime): + return value.isoformat().replace('+00:00', 'Z') + return value + +CUTOFF = datetime(2026, 2, 1, tzinfo=timezone.utc) +DAY_SECONDS = 86400 +DAY_US = DAY_SECONDS * 1000000 # Workload duration fields retain their schema units. +USER_A = 9007199254740993 +USER_B = -5 + + +def column(dtype, nullable=False, stype=None): + spec = {"dtype": dtype, "nullable": nullable} + if stype: + spec["stype"] = stype + return spec + + +def table(pk, columns, time_column=None): + return {"primary_key": pk, "columns": columns, "time_column": time_column, + **({"availability": "timeless"} if time_column is None else {})} + + +def commerce(): + tables = { + "users": table("user_id", {"user_id": column("int64"), "segment": column("string", stype="categorical"), "active": column("bool")}), + "interactions": table("interaction_id", {"interaction_id": column("int64"), "user_id": column("int64", True), + "item_id": column("string", True), "event_time": column("timestamp[us]", True), "amount": column("float64", True, "numerical")}, "event_time"), + "items": table("item_id", {"item_id": column("string"), "category_id": column("int64"), "description": column("string", stype="text")}), + "categories": table("category_id", {"category_id": column("int64"), "name": column("string", stype="categorical")}), + } + relations = {"interaction_user": {"child": "interactions", "foreign_key": "user_id", "parent": "users"}, + "interaction_item": {"child": "interactions", "foreign_key": "item_id", "parent": "items"}, + "item_category": {"child": "items", "foreign_key": "category_id", "parent": "categories"}} + events = [(11, USER_A, "shared", -1), (12, USER_A, "shared", -1), + (13, USER_A, "other", -1), (21, USER_B, "shared", -2), + (22, USER_B, "other", -10 * DAY_SECONDS), (23, USER_B, "other", -10 * DAY_SECONDS - 1), + (30, USER_A, "other", 0), (31, USER_A, "shared", DAY_SECONDS), + (40, USER_A, None, -3), (41, None, "shared", -1), + (42, USER_B, "other", None), (43, USER_A, "shared", -40 * DAY_SECONDS - 1), + (44, USER_A, "other", -20 * DAY_SECONDS)] + rows = { + "users": [{"user_id": USER_A, "segment": "a", "active": True}, + {"user_id": USER_B, "segment": "b", "active": False}, + {"user_id": 42, "segment": "quiet", "active": True}], + "interactions": [{"interaction_id": k, "user_id": u, "item_id": item, + "event_time": None if dt is None else CUTOFF + timedelta(seconds=dt), + "amount": None if k == 22 else 5.0} for k, u, item, dt in events], + "items": [{"item_id": "shared", "category_id": 7, "description": "shared item"}, + {"item_id": "other", "category_id": 7, "description": "other item"}, + {"item_id": "unused", "category_id": 8, "description": "unused"}], + "categories": [{"category_id": 7, "name": "common"}, {"category_id": 8, "name": "unused"}], + } + layers = [[{"relation": "interaction_user", "direction": "reverse", "fanout": 2}], + [{"relation": "interaction_item", "direction": "forward", "fanout": 1}], + [{"relation": "item_category", "direction": "forward", "fanout": 1}]] + return tables, relations, rows, "users", "interactions", "user_id", "event_time", layers + + +def marketo(): + facts = {"engagements": ("marketo_lead_id", "engagement_datetime"), + "metrics": ("marketo_lead_id", "trigger_date"), + "usecase_metrics": ("marketo_lead_id", "trigger_date"), + "conversions": ("marketo_lead_id", "u2_stage_date"), + "activities": ("id", "activity_date"), + "notes": ("marketo_lead_id", "source_creation_timestamp")} + tables = {"leads": table("Id", {"Id": column("int64"), "segment": column("string", stype="categorical")})} + rows = {"leads": [{"Id": USER_A, "segment": "a"}, {"Id": USER_B, "segment": "b"}, {"Id": 42, "segment": "quiet"}]} + relations, layer = {}, [] + for name, (fk, tc) in facts.items(): + tables[name] = table("event_id", {"event_id": column("int64"), fk: column("int64", True), + tc: column("timestamp[us]", True), "value": column("float64", True, "numerical")}, tc) + rows[name] = [] if name == "notes" else [ + {"event_id": k, fk: u, tc: CUTOFF + timedelta(seconds=dt), "value": None if k == 3 else 1.0} + for k, u, dt in [(1, USER_A, -1), (2, USER_A, -1), (3, USER_A, -1), + (4, USER_B, -10 * DAY_SECONDS), (5, USER_B, 0), (6, None, -1), + (7, USER_A, -20 * DAY_SECONDS)]] + relations[name + "_lead"] = {"child": name, "foreign_key": fk, "parent": "leads"} + layer.append({"relation": name + "_lead", "direction": "reverse", "fanout": 2}) + return tables, relations, rows, "leads", "engagements", "marketo_lead_id", "engagement_datetime", [layer] + + +def write_bundle(root, tables, relations, rows, tasks, *, recipe, reverse_layout=False, row_group_size=3): + """Small-fixture writer; imported/bulk bundles use the documented same format.""" + root = Path(root) + root.mkdir(parents=True, exist_ok=False) + inventory = {} + logical_tables = {} + for name, spec in tables.items(): + path = root / "tables" / name / "part-00000.parquet" + path.parent.mkdir(parents=True) + schema = pa.schema([pa.field(c, TYPES[s["dtype"]], s["nullable"]) for c, s in spec["columns"].items()]) + data = list(reversed(rows[name])) if reverse_layout else rows[name] + pq.write_table(pa.Table.from_pylist(data, schema=schema), path, row_group_size=row_group_size) + relative = str(path.relative_to(root)) + spec.update(rows=len(data), files=[relative]) + ordered = sorted(rows[name], key=lambda r: r[spec["primary_key"]]) + logical_tables[name] = json.loads(json.dumps(ordered, default=scalar)) + meta = pq.ParquetFile(path).metadata + inventory[relative] = {"sha256": digest(path), "rows": len(data), "bytes": path.stat().st_size, + "row_groups": [meta.row_group(i).num_rows for i in range(meta.num_row_groups)]} + task_specs = {} + for name, task in tasks.items(): + task_specs[name] = {k: v for k, v in task.items() if k not in ("context", "predict", "labels")} + entity = tables[task["entity_table"]] + key_type = TYPES[entity["columns"][entity["primary_key"]]["dtype"]] + for split in ("context", "predict", "labels"): + path = root / "tasks" / name / f"{split}.parquet" + path.parent.mkdir(parents=True, exist_ok=True) + if split == "labels": + schema = pa.schema([pa.field("instance_id", pa.int64(), False), pa.field("target", TYPES[task["target"]["dtype"]], False)]) + else: + fields = [pa.field("instance_id", pa.int64(), False), pa.field("entity_key", key_type, False), + pa.field("anchor_time", TYPES["timestamp[us]"], False)] + if split == "context": + fields += [pa.field("target", TYPES[task["target"]["dtype"]], False), pa.field("label_available_at", TYPES["timestamp[us]"], False)] + schema = pa.schema(fields) + pq.write_table(pa.Table.from_pylist(task[split], schema=schema), path) + relative = str(path.relative_to(root)) + inventory[relative] = {"sha256": digest(path), "rows": len(task[split]), "bytes": path.stat().st_size} + task_specs[name][split] = relative + logical = {"recipe": recipe, "tables": logical_tables, + "schema": {n: {k: v for k, v in t.items() if k not in ("files", "rows")} for n, t in tables.items()}, + "relations": relations} + manifest = {"format_version": 2, "logical_id": identity(logical), "recipe": recipe, + "tables": tables, "relations": relations, "tasks": task_specs, "files": inventory, + "writer": {"pyarrow": pa.__version__, "row_group_size": row_group_size, "reverse_layout": reverse_layout}} + (root / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + # Perform real validation before publishing the completion marker. + provisional = Bundle.__new__(Bundle) + provisional.root, provisional.manifest = root.resolve(), manifest + provisional.tables, provisional.relations = tables, relations + report = {"status": "passed", **provisional.validate()} + (root / "validation.json").write_text(json.dumps(report, indent=2) + "\n") + (root / "_COMPLETE.json").write_text(json.dumps({"manifest_sha256": digest(root / "manifest.json"), + "validation_sha256": digest(root / "validation.json")}, indent=2) + "\n") + return Bundle(root) + + +def generate(root: Path, preset="commerce_v1", *, reverse_layout=False, row_group_size=3): + if preset not in ("commerce_v1", "marketo_v1"): + raise ValueError("Choose commerce_v1 or marketo_v1") + tables, relations, rows, entity, fact, fk, tc, layers = commerce() if preset == "commerce_v1" else marketo() + tasks = {} + context = [{"instance_id": 0, "entity_key": USER_A, "anchor_time": CUTOFF - timedelta(days=40)}, + {"instance_id": 1, "entity_key": USER_B, "anchor_time": CUTOFF - timedelta(days=40)}] + predict = [{"instance_id": i, "entity_key": key, "anchor_time": CUTOFF} + for i, key in [(100, USER_A), (101, USER_B), (102, USER_A), (103, 42)]] + def count(seed): + return sum(1 for r in rows[fact] if r[fk] == seed["entity_key"] and r[tc] is not None + and seed["anchor_time"] <= r[tc] < seed["anchor_time"] + timedelta(days=30)) + for kind, target in [("binary_classification", {"dtype": "string", "classes": ["no", "yes"]}), + ("regression", {"dtype": "int64"})]: + def label(seed): + return count(seed) if kind == "regression" else ("yes" if count(seed) else "no") + tasks[kind] = {"entity_table": entity, "kind": kind, "target": target, + "label_horizon_us": 30 * DAY_US, + "context": [{**s, "target": label(s), "label_available_at": s["anchor_time"] + timedelta(days=30)} for s in context], + "predict": predict, "labels": [{"instance_id": s["instance_id"], "target": label(s)} for s in predict]} + bundle = write_bundle(root, tables, relations, rows, tasks, + recipe={"generator": "tiny_contract_v3", "topology": preset, + "seed": 42, "timestamp_granularity": "seconds"}, + reverse_layout=reverse_layout, row_group_size=row_group_size) + workload_dir = Path(root) / "workloads" + workload_dir.mkdir() + hops = (2, 3) if preset == "commerce_v1" else (1,) + for kind in tasks: + for depth in hops: + selected = layers[:depth] + names = {entity} + for layer in selected: + for step in layer: + names.update(bundle.endpoint(step)) + spec = {"version": 1, "logical_id": bundle.logical_id, "snapshot_id": bundle.snapshot_id, + "task": kind, "prediction_cutoff": scalar(CUTOFF), + "selection": "local_native", "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", "layers": selected, + "projections": {n: list(tables[n]["columns"]) for n in sorted(names)}, "encoding": "arrays"} + (workload_dir / f"{kind}-{depth}hop.json").write_text(json.dumps(spec, indent=2) + "\n") + return bundle diff --git a/packages/experimental/kumo-fm/tests/test_batch.py b/packages/experimental/kumo-fm/tests/test_batch.py new file mode 100644 index 000000000..2410d7ac5 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_batch.py @@ -0,0 +1,90 @@ +import base64 +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +import pyarrow as pa +import pytest + +from kumo.fm.batch import PayloadBatch + + +BIG_ID = 9007199254740993 + + +def workload(): + tables = { + "users": {"primary_key": "user_id", "columns": { + "user_id": {"dtype": "int64", "nullable": False}, + "segment": {"dtype": "string", "nullable": True}}}, + "events": {"primary_key": "event_id", "columns": { + "event_id": {"dtype": "int64", "nullable": False}, + "user_id": {"dtype": "int64", "nullable": True}}}, + } + bundle = SimpleNamespace(tables=tables, relations={ + "event_user": {"child": "events", "foreign_key": "user_id", "parent": "users"}}) + return SimpleNamespace( + bundle=bundle, entity="users", table_names=set(tables), relation_names={"event_user"}, + projections={"users": ["segment"], "events": ["event_id", "user_id"]}, + target={"dtype": "int64", "classes": [0, 1]}, kind="classification", + budgets={"max_columns": 10, "max_rows_per_table": 10, + "max_decoded_arrow_bytes": 100000, "max_payload_bytes": 100000}) + + +def request(work): + anchor = datetime(2026, 2, 1, tzinfo=timezone.utc) + context = [{"instance_id": 7, "entity_key": BIG_ID, "anchor_time": anchor, "target": 1}] + predict = [{"instance_id": 8, "entity_key": BIG_ID, "anchor_time": anchor}] + def related(instance_id): + return { + "users": pa.table({"instance_id": pa.array([instance_id], pa.int64()), + "user_id": pa.array([BIG_ID], pa.int64()), + "segment": pa.array(["a"])}), + "events": pa.table({"instance_id": pa.array([instance_id], pa.int64()), + "event_id": pa.array([17], pa.int64()), + "user_id": pa.array([BIG_ID], pa.int64()), + "__fm_edge_event_user": pa.array([BIG_ID], pa.int64())}), + } + return PayloadBatch(work, {"context": (context, related(7)), + "predict": (predict, related(8))}) + + +def test_retained_tables_encode_both_formats_without_mutation(): + batch = request(workload()) + before = batch.tables("context")["related_tables"]["events"] + metadata = batch.metadata + metadata["task"]["kind"] = "changed" + assert batch.metadata["task"]["kind"] == "classification" + direct = batch.as_arrow_request() + arrays = json.loads(batch.encode("arrays")) + ipc = json.loads(batch.encode("arrow_ipc")) + assert arrays["context"]["related_tables"]["events"]["rows"] == [[7, 17, str(BIG_ID), str(BIG_ID)]] + for split in ("context", "predict"): + for name in ("users", "events"): + encoded = ipc[split]["related_tables"][name] + decoded = pa.ipc.open_stream(base64.b64decode(encoded["payload_base64"])).read_all() + assert decoded.equals(direct[split]["related_tables"][name]) + assert batch.tables("context")["related_tables"]["events"].equals(before) + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + assert batch.logical_arrow_bytes == sum( + frame.nbytes for split in batch.splits for frame in ( + batch.tables(split)["instance_table"], + *batch.tables(split)["related_tables"].values())) + with pytest.raises(ValueError, match="Unknown encoding"): + batch.encode("invalid") + assert batch.encode("arrays") + + +def test_logical_and_wire_budgets_are_independent(): + work = workload() + work.budgets["max_decoded_arrow_bytes"] = 1 + with pytest.raises(ValueError, match="Decoded Arrow budget"): + request(work) + work = workload() + work.budgets["max_payload_bytes"] = 1 + batch = request(work) + assert batch.tables("context")["instance_table"].num_rows == 1 + with pytest.raises(ValueError, match="byte budget"): + batch.encode("arrays") + with pytest.raises(ValueError, match="byte budget"): + batch.encode("arrow_ipc") diff --git a/packages/experimental/kumo-fm/tests/test_contract.py b/packages/experimental/kumo-fm/tests/test_contract.py new file mode 100644 index 000000000..f5091872b --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_contract.py @@ -0,0 +1,31 @@ +from types import SimpleNamespace + +import pytest + +from kumo.fm._pytorch import compatibility_issues, contract_for +from kumo.fm.loader import _select + + +def test_unsupported_sampling_is_explicit(): + bundle = SimpleNamespace( + tables={"events": {"time_column": None}}, + endpoint=lambda step: ("users", "events")) + workload = SimpleNamespace( + bundle=bundle, selection="native_multihop", + layers=[[{"relation": "event_user", "direction": "reverse", "fanout": 3}]]) + assert "uniform sampling" in compatibility_issues(workload)[0] + with pytest.raises(ValueError, match="Cannot preserve FM sampling"): + contract_for(workload) + workload.layers = [[{"relation": "event_user", "direction": "reverse", "fanout": 0}]] + assert "empty graph layer" in compatibility_issues(workload)[0] + workload.selection = "temporal_query" + assert "native_multihop" in compatibility_issues(workload)[0] + + +def test_anchor_selection_keeps_order_and_rejects_unknown_ids(): + available = [{"instance_id": 1}, {"instance_id": 2}] + assert _select(available, [2, 1], "predict") == [available[1], available[0]] + with pytest.raises(ValueError, match="duplicate or unknown"): + _select(available, [1, 1], "predict") + with pytest.raises(ValueError, match="duplicate or unknown"): + _select(available, [3], "predict") diff --git a/packages/experimental/kumo-fm/tests/test_integration.py b/packages/experimental/kumo-fm/tests/test_integration.py new file mode 100644 index 000000000..bf323ad09 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_integration.py @@ -0,0 +1,63 @@ +import json +from copy import deepcopy +from datetime import timedelta + +import pyarrow.parquet as pq + +from kumo import fm + +from _fixture import CUTOFF, USER_A, USER_B, generate, write_bundle + + +def fixture(tmp_path): + original = generate(tmp_path / "original") + tables = deepcopy(original.tables) + rows = {name: pq.read_table(original.paths(name)).to_pylist() for name in tables} + rows["interactions"] = [ + {"interaction_id": event_id, "user_id": user, "item_id": "shared", + "event_time": CUTOFF + timedelta(days=days), "amount": 5.0} + for event_id, user, days in ((1, USER_A, -41), (2, USER_A, -1), + (3, USER_B, -41), (4, USER_B, -1)) + ] + tasks = deepcopy(original.manifest["tasks"]) + for task in tasks.values(): + for split in ("context", "predict", "labels"): + task[split] = pq.read_table(original.path(task[split])).to_pylist() + bundle = write_bundle(tmp_path / "supported", tables, deepcopy(original.relations), + rows, tasks, recipe={"generator": "kumo_fm_integration"}) + spec = json.loads((original.root / "workloads/binary_classification-2hop.json").read_text()) + spec.update(logical_id=bundle.logical_id, snapshot_id=bundle.snapshot_id, + selection="native_multihop", layers=spec["layers"][:1], + projections={name: list(bundle.tables[name]["columns"]) + for name in ("users", "interactions")}) + spec["layers"][0][0]["fanout"] = 1 + return bundle, spec + + +def test_installed_wheel_samples_once_and_batch_survives_close(tmp_path): + bundle, spec = fixture(tmp_path) + index = tmp_path / "index" + with fm.prepare(bundle, spec, index=index) as loader: + assert loader.preparation_mode == "index_build" + with loader.materialize() as request: + batch = request.load() + loader._backend.extract = lambda *_: (_ for _ in ()).throw( + AssertionError("encoding resampled")) + arrays = batch.encode("arrays") + ipc = batch.encode("arrow_ipc") + assert arrays and ipc + assert batch.tables("predict")["instance_table"].num_rows == 4 + context_events = batch.tables("context")["related_tables"]["interactions"] + query_events = batch.tables("predict")["related_tables"]["interactions"] + assert set(context_events["interaction_id"].to_pylist()) == {1, 3} + assert set(query_events["interaction_id"].to_pylist()) == {2, 4} + assert set(zip(query_events["instance_id"].to_pylist(), + query_events["interaction_id"].to_pylist())) == { + (100, 2), (101, 4), (102, 2)} + assert all(value is not None for value in query_events["__fm_edge_interaction_user"].to_pylist()) + with fm.prepare(bundle, spec, index=index) as loader: + assert loader.preparation_mode == "cached_index_open" + with loader.materialize(context=[], predict=[100]) as request: + single = request.load() + assert single.splits == ("predict",) + assert single.tables("predict")["instance_table"].num_rows == 1 diff --git a/packages/experimental/kumo-fm/tests/test_sdm_adapter.py b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py new file mode 100644 index 000000000..563ff39c3 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py @@ -0,0 +1,63 @@ +import sys +from pathlib import Path +from types import SimpleNamespace + +import pyarrow as pa + +from test_batch import request, workload + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "examples/local_sdm")) +from adapter import _compatible_buffers, arrow_to_sdm + + +def test_sdm_bridge_uses_public_arrow_conversion(monkeypatch): + calls = [] + + class TableTensor: + def __init__(self, table, stypes): + self.table, self.stypes = table, stypes + + @classmethod + def from_arrow(cls, table, *, stypes, device): + assert device == "cpu" + calls.append(table) + return cls(table, stypes) + + def drop_columns(self, name): + return TableTensor(self.table.drop([name]), { + key: value for key, value in self.stypes.items() if key != name}) + + def __getitem__(self, names): + return TableTensor(self.table.select(names), { + key: self.stypes[key] for key in names}) + + class RelatedTables: + def __init__(self, *, tables, relationships, task_links): + self.tables, self.relationships, self.task_links = tables, relationships, task_links + + monkeypatch.setitem(sys.modules, "sdm", SimpleNamespace( + TableTensor=TableTensor, RelatedTables=RelatedTables)) + batch = request(workload()) + source = batch.tables("context")["related_tables"]["events"] + result = arrow_to_sdm(batch) + assert len(calls) == 6 + assert source.equals(batch.tables("context")["related_tables"]["events"]) + assert result["y_context"].stypes == {"__fm_target": "categorical"} + assert "__fm_target" not in result["x_query"].stypes + assert result["related_context_tables"].relationships[0]["left_columns"] == [ + "instance_id", "__fm_edge_event_user"] + assert result["related_query_tables"].task_links[0]["task_columns"] == [ + "instance_id", "__fm_entity_key"] + + +def test_buffer_compatibility_only_repairs_incompatible_columns(): + writable = pa.table({"name": pa.array(["a", None], type=pa.string())}) + assert _compatible_buffers(writable) is writable + sink = pa.BufferOutputStream() + with pa.ipc.new_stream(sink, writable.schema) as writer: + writer.write_table(writable) + read_only = pa.ipc.open_stream(sink.getvalue()).read_all() + assert not read_only["name"].chunk(0).buffers()[2].is_mutable + repaired = _compatible_buffers(read_only) + assert repaired.equals(read_only) + assert repaired["name"].chunk(0).buffers()[2].is_mutable diff --git a/pyproject.toml b/pyproject.toml index 965fa9195..1000a73dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ members = [ "packages/kumo-engine", "packages/kumo-sdk", "packages/experimental/kumo-pytorch", + "packages/experimental/kumo-fm", "tools/glove-6b-50d", ] diff --git a/uv.lock b/uv.lock index f46d2a930..af21516ca 100644 --- a/uv.lock +++ b/uv.lock @@ -57,6 +57,7 @@ members = [ "kumo-core", "kumo-databricks", "kumo-engine", + "kumo-fm", "kumo-gcs", "kumo-glove-6b-50d", "kumo-pql", @@ -1601,6 +1602,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408, upload-time = "2024-04-23T18:57:14.835Z" }, ] +[[package]] +name = "duckdb" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +requires-python = ">=3.11.0" +sdist = { url = "https://files.pythonhosted.org/packages/c2/7c/dfdf5e9cebefd032f57f09b224993e861ecb55f61e36c709544b04b44f70/duckdb-2.0.0.tar.gz", hash = "sha256:09de511ef0dc76a5213e8d3d87a8b89bb4d3e8a6fe73a9f60dc001269946b44f", size = 16771077, upload-time = "2026-10-21T12:51:13.908572Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/57/ec7c1a01ab43fc13c55c17cc54a5740886e04ae830aa420ac00614b1420a/duckdb-2.0.0-cp311-cp311-macosx_10_15_universal2.whl", hash = "sha256:a3b38d8cdbef10007c64b65efc4087f799c4dc606785aa5f2a425ed13658ca01", size = 37006608, upload-time = "2026-10-21T12:50:38.611878Z" }, + { url = "https://files.pythonhosted.org/packages/84/36/e1ea89acd74917ca654d5adc278765f0116e07540abf4134d5cf9f527363/duckdb-2.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:9bcc9921be0e1b2404b96a7fb447a8b80fe2f3a919cdc976ca6c39951cc8f2f5", size = 19691094, upload-time = "2026-10-21T12:50:41.839172Z" }, + { url = "https://files.pythonhosted.org/packages/4b/81/2895b9ee3858dae7a57f791d7dbd4f116e9fa8fd3c71b9a377dbd4d292a7/duckdb-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:61a244faae9fd787e30f4c76888769ec49e79736c434f698e8178a00af8348f0", size = 17603089, upload-time = "2026-10-21T12:50:44.167966Z" }, + { url = "https://files.pythonhosted.org/packages/04/95/0c8b5b95ac9e14ee83090a78048fbb23fb9f8abacd63e11a5eee7fcb31e1/duckdb-2.0.0-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcdf10dfbc2c1a6bdc5b2a94bba5b15481e552c5745c2af1994617b0eb6bc750", size = 21637946, upload-time = "2026-10-21T12:50:46.444674Z" }, + { url = "https://files.pythonhosted.org/packages/e4/a1/9c3780faada6e6a3edc5f9364cf19810e6e1ea90a7d5fd8fffc48b42c269/duckdb-2.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a3601121c7cc50d0b70412995634682fafcb58a0156d5d663dfd0b3bbe30a33", size = 24351483, upload-time = "2026-10-21T12:50:48.902652Z" }, + { url = "https://files.pythonhosted.org/packages/b5/01/0729ac5d9857e5f0bde7262961f80e578a29945a444169886094cc9c69f4/duckdb-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a387d4907b535a4bd82ba8aede6809d0bf023dfbb21362832f3c9848aea7743c", size = 15370691, upload-time = "2026-10-21T12:50:51.590941Z" }, + { url = "https://files.pythonhosted.org/packages/09/13/392991ec9dcb22811c7029009184a910bd5cb69fbb7a626f36f151215535/duckdb-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:62bea9a1786be98d2906197c67a4089cc8b34d66d19e87e5543d8201e81323ad", size = 15731407, upload-time = "2026-10-21T12:50:53.889460Z" }, + { url = "https://files.pythonhosted.org/packages/41/28/1fb52f9ea10205a55a07c32e37cd4fd68fa9d1dc2520a12405ec9bb59cac/duckdb-2.0.0-cp312-abi3-macosx_10_15_universal2.whl", hash = "sha256:cb1cc30318397c3bcffa6b683196293698052588de3c1ba1e144ba0c72560ce9", size = 37003294, upload-time = "2026-10-21T12:50:56.489773Z" }, + { url = "https://files.pythonhosted.org/packages/07/5b/fbe92644709fb9162cc51753f92f892b1f333658079d1f11efaed43341d2/duckdb-2.0.0-cp312-abi3-macosx_10_15_x86_64.whl", hash = "sha256:46c7a299ff5f45be9068cfc0ceecb3fd574af9535de8cb6908ae8b52d2846cbd", size = 19689207, upload-time = "2026-10-21T12:50:59.331542Z" }, + { url = "https://files.pythonhosted.org/packages/8f/a3/4d23938223626980cdcf18fd6b0dbf040823b5cf15beabbf4ab7ee276631/duckdb-2.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:b5d758b53751c930e09d371230241005ea5203c8edc65a1e1fafca83fcc906ef", size = 17597612, upload-time = "2026-10-21T12:51:01.818477Z" }, + { url = "https://files.pythonhosted.org/packages/92/4a/985fb9d93d7f9f8e82092ddfe05ec8c9a1d0df7bdb0f562a10ced30e3e8f/duckdb-2.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e38daf0acbad9de7ce892d402b68409519291ee16af3c2cba85ace3b085c7eaa", size = 21655920, upload-time = "2026-10-21T12:51:04.360238Z" }, + { url = "https://files.pythonhosted.org/packages/cc/33/01ad2535ee897db2bad70cbc800655167c53afd0b955eed84f0759c2f417/duckdb-2.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ab51ec57d329d6f3df37c0287b55d71c3fc07bb54b61d1afed09104f953b1431", size = 24371005, upload-time = "2026-10-21T12:51:06.860701Z" }, + { url = "https://files.pythonhosted.org/packages/09/d3/9e8ca1242b98f04041bef09686ecfe098b41f820aa23240f63f360b64eaf/duckdb-2.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:f61c96547ed432f54dae42a717fd88563eda55cb47549715fac9f61f770c205a", size = 15368860, upload-time = "2026-10-21T12:51:09.395345Z" }, + { url = "https://files.pythonhosted.org/packages/37/81/6ea5dc1050f86ec4dd038a2774766b42247ba6104e7785f13a0c600e2b33/duckdb-2.0.0-cp312-abi3-win_arm64.whl", hash = "sha256:c5e6708efaa8647e0914a16f1914354419e529a1925929950e7d610418e45873", size = 15726489, upload-time = "2026-10-21T12:51:11.643301Z" }, +] + [[package]] name = "ec2-metadata" version = "2.10.0" @@ -3705,6 +3729,36 @@ requires-dist = [ ] provides-extras = ["test", "docs"] +[[package]] +name = "kumo-fm" +version = "0.1.0a1" +source = { editable = "packages/experimental/kumo-fm" } +dependencies = [ + { name = "duckdb" }, + { name = "kumo-pytorch" }, + { name = "numpy" }, + { name = "pyarrow" }, +] + +[package.optional-dependencies] +relbench = [ + { name = "pyyaml" }, +] +test = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "duckdb", specifier = "==2.0.0" }, + { name = "kumo-pytorch", editable = "packages/experimental/kumo-pytorch", specifier = ">=0.1.0a1" }, + { name = "numpy", specifier = ">=1.26,<3" }, + { name = "pyarrow", specifier = "==24.0.0" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8,<10" }, + { name = "pyyaml", marker = "extra == 'relbench'", specifier = ">=6,<7" }, +] +provides-extras = ["relbench", "test"] + [[package]] name = "kumo-pytorch" source = { editable = "packages/experimental/kumo-pytorch" } From 5960636b26ffd98b49b14fd0163e79cb8b8e709a Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Wed, 18 Nov 2026 18:52:27 +0000 Subject: [PATCH 03/12] Include runnable FM SDM example fixture --- benchmark/fm-payloads/README.md | 2 +- benchmark/fm-payloads/make_fixture.py | 5 +- packages/experimental/kumo-fm/MANIFEST.in | 1 + .../kumo-fm/examples/local_sdm/README.md | 14 +- .../local_sdm/fixture/bundle/_COMPLETE.json | 4 + .../local_sdm/fixture/bundle/manifest.json | 223 ++++++++++++++++++ .../tables/categories/part-00000.parquet | Bin 0 -> 755 bytes .../tables/interactions/part-00000.parquet | Bin 0 -> 2730 bytes .../bundle/tables/items/part-00000.parquet | Bin 0 -> 1056 bytes .../bundle/tables/users/part-00000.parquet | Bin 0 -> 913 bytes .../binary_classification/context.parquet | Bin 0 -> 1689 bytes .../binary_classification/labels.parquet | Bin 0 -> 748 bytes .../binary_classification/predict.parquet | Bin 0 -> 1148 bytes .../bundle/tasks/regression/context.parquet | Bin 0 -> 1750 bytes .../bundle/tasks/regression/labels.parquet | Bin 0 -> 808 bytes .../bundle/tasks/regression/predict.parquet | Bin 0 -> 1148 bytes .../local_sdm/fixture/bundle/validation.json | 31 +++ .../examples/local_sdm/fixture/workload.json | 34 +++ .../examples/local_sdm/walkthrough.ipynb | 4 +- 19 files changed, 307 insertions(+), 11 deletions(-) create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/interactions/part-00000.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md index 6db60cf8b..d0dfc1502 100644 --- a/benchmark/fm-payloads/README.md +++ b/benchmark/fm-payloads/README.md @@ -9,7 +9,7 @@ Run correctness first, then warmups and interleaved measured draws: ```bash python make_fixture.py /tmp/fm-case -python runner.py --bundle /tmp/fm-case/supported \ +python runner.py --bundle /tmp/fm-case/bundle \ --workload /tmp/fm-case/workload.json --index /tmp/fm-case/index \ --draws 5 --output /tmp/fm-report.json ``` diff --git a/benchmark/fm-payloads/make_fixture.py b/benchmark/fm-payloads/make_fixture.py index 08de153e7..61cdd0b92 100644 --- a/benchmark/fm-payloads/make_fixture.py +++ b/benchmark/fm-payloads/make_fixture.py @@ -15,9 +15,10 @@ def main() -> None: raise SystemExit("usage: python make_fixture.py /path/to/new/case") root = Path(sys.argv[1]).resolve() root.mkdir(parents=True, exist_ok=False) - bundle, workload = fixture(root) + _, workload = fixture(root) + (root / "supported").rename(root / "bundle") (root / "workload.json").write_text(json.dumps(workload, indent=2) + "\n") - print(f"bundle: {bundle.root}") + print(f"bundle: {root / 'bundle'}") print(f"workload: {root / 'workload.json'}") diff --git a/packages/experimental/kumo-fm/MANIFEST.in b/packages/experimental/kumo-fm/MANIFEST.in index f44b7b194..f87b568a3 100644 --- a/packages/experimental/kumo-fm/MANIFEST.in +++ b/packages/experimental/kumo-fm/MANIFEST.in @@ -1,2 +1,3 @@ recursive-include examples/local_sdm *.py *.ipynb *.md *.lock +recursive-include examples/local_sdm/fixture *.json *.parquet recursive-include tests *.py diff --git a/packages/experimental/kumo-fm/examples/local_sdm/README.md b/packages/experimental/kumo-fm/examples/local_sdm/README.md index cd96be487..8d647b085 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/README.md +++ b/packages/experimental/kumo-fm/examples/local_sdm/README.md @@ -7,18 +7,20 @@ from one engine revision, then the pinned SDM environment in `nvidia/Kumo-Relational` v1.0.1 checkpoint; the prior validation used snapshot `2bd603d3d8f25f67a7aaa8579908595e20567e22`. -Set `FM_SDM_FIXTURE` to a directory containing `bundle/` and -`workload.json`. The workload must be an eligible `native_multihop` -contract. Set `FM_SDM_BUNDLE` when the bundle has another path. The notebook opens or creates `fm-index/` there, samples one +The example includes `fixture/bundle/` and `fixture/workload.json`. Run the +notebook from this directory to use them by default. Set `FM_SDM_FIXTURE` to +another directory containing `bundle/` and `workload.json` to use your own +eligible `native_multihop` workload. Set `FM_SDM_BUNDLE` when its bundle has +another path. The notebook opens or creates `fm-index/` in the fixture directory, samples one batch, converts its retained Arrow tables through public `TableTensor.from_arrow` and `RelatedTables`, and exercises both model call forms. The helper only repairs zero-chunk columns and read-only string buffers when SDM requires it. The same batch is encoded as arrays and IPC. -The committed smoke case can be recreated with +Regenerate the included smoke case with `python benchmark/fm-payloads/make_fixture.py /tmp/fm-case` from the repository -root. Set `FM_SDM_FIXTURE=/tmp/fm-case` and -`FM_SDM_BUNDLE=/tmp/fm-case/supported` before running `walkthrough.ipynb`. +root, then copy `bundle/` and `workload.json` into this directory's `fixture/`. +You can also set `FM_SDM_FIXTURE=/tmp/fm-case` to use the generated copy. The notebook includes the local execution output: both model call forms agreed with a maximum absolute difference of 0.0 on this fixture. diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json new file mode 100644 index 000000000..81c44a755 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/_COMPLETE.json @@ -0,0 +1,4 @@ +{ + "manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", + "validation_sha256": "359d774e6cbd42a0b3652022842dc2e228e8488527b6a9ee3e87352dc7a40003" +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json new file mode 100644 index 000000000..ad24cc00c --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/manifest.json @@ -0,0 +1,223 @@ +{ + "format_version": 2, + "logical_id": "b1b964d2a964e4fb95b4cb093d5de467199f2b10b67970e8fe3b5407d7b8137a", + "recipe": { + "generator": "kumo_fm_integration" + }, + "tables": { + "users": { + "primary_key": "user_id", + "columns": { + "user_id": { + "dtype": "int64", + "nullable": false + }, + "segment": { + "dtype": "string", + "nullable": false, + "stype": "categorical" + }, + "active": { + "dtype": "bool", + "nullable": false + } + }, + "time_column": null, + "availability": "timeless", + "rows": 3, + "files": [ + "tables/users/part-00000.parquet" + ] + }, + "interactions": { + "primary_key": "interaction_id", + "columns": { + "interaction_id": { + "dtype": "int64", + "nullable": false + }, + "user_id": { + "dtype": "int64", + "nullable": true + }, + "item_id": { + "dtype": "string", + "nullable": true + }, + "event_time": { + "dtype": "timestamp[us]", + "nullable": true + }, + "amount": { + "dtype": "float64", + "nullable": true, + "stype": "numerical" + } + }, + "time_column": "event_time", + "rows": 4, + "files": [ + "tables/interactions/part-00000.parquet" + ] + }, + "items": { + "primary_key": "item_id", + "columns": { + "item_id": { + "dtype": "string", + "nullable": false + }, + "category_id": { + "dtype": "int64", + "nullable": false + }, + "description": { + "dtype": "string", + "nullable": false, + "stype": "text" + } + }, + "time_column": null, + "availability": "timeless", + "rows": 3, + "files": [ + "tables/items/part-00000.parquet" + ] + }, + "categories": { + "primary_key": "category_id", + "columns": { + "category_id": { + "dtype": "int64", + "nullable": false + }, + "name": { + "dtype": "string", + "nullable": false, + "stype": "categorical" + } + }, + "time_column": null, + "availability": "timeless", + "rows": 2, + "files": [ + "tables/categories/part-00000.parquet" + ] + } + }, + "relations": { + "interaction_user": { + "child": "interactions", + "foreign_key": "user_id", + "parent": "users" + }, + "interaction_item": { + "child": "interactions", + "foreign_key": "item_id", + "parent": "items" + }, + "item_category": { + "child": "items", + "foreign_key": "category_id", + "parent": "categories" + } + }, + "tasks": { + "binary_classification": { + "entity_table": "users", + "kind": "binary_classification", + "target": { + "dtype": "string", + "classes": [ + "no", + "yes" + ] + }, + "label_horizon_us": 2592000000000, + "context": "tasks/binary_classification/context.parquet", + "predict": "tasks/binary_classification/predict.parquet", + "labels": "tasks/binary_classification/labels.parquet" + }, + "regression": { + "entity_table": "users", + "kind": "regression", + "target": { + "dtype": "int64" + }, + "label_horizon_us": 2592000000000, + "context": "tasks/regression/context.parquet", + "predict": "tasks/regression/predict.parquet", + "labels": "tasks/regression/labels.parquet" + } + }, + "files": { + "tables/users/part-00000.parquet": { + "sha256": "3bbcc77313b1cfc069b64512b7b1ff84b006dea5785df1822dc7e839bc522154", + "rows": 3, + "bytes": 913, + "row_groups": [ + 3 + ] + }, + "tables/interactions/part-00000.parquet": { + "sha256": "0e7fed483ac455b984f28b46438ef4d4de8549f30c8c8eb134cf8f24a8baffb5", + "rows": 4, + "bytes": 2730, + "row_groups": [ + 3, + 1 + ] + }, + "tables/items/part-00000.parquet": { + "sha256": "baab8536005f41365a3b347c1051ebae1437665d399600a758d5497eb2ed0474", + "rows": 3, + "bytes": 1056, + "row_groups": [ + 3 + ] + }, + "tables/categories/part-00000.parquet": { + "sha256": "b8f5211b06fd6cb5cba9b4c5680cadfa775920b571ea21f6fd36d8b37cd3faf8", + "rows": 2, + "bytes": 755, + "row_groups": [ + 2 + ] + }, + "tasks/binary_classification/context.parquet": { + "sha256": "5e13ecd914bca54712a062c191ffaff326998d1b93e03a11ce66753d06e4603d", + "rows": 2, + "bytes": 1689 + }, + "tasks/binary_classification/predict.parquet": { + "sha256": "2fcf0a7d39150dc136caf5ddf06c51156eb0e147dfa50c6914141eb78f3dc256", + "rows": 4, + "bytes": 1148 + }, + "tasks/binary_classification/labels.parquet": { + "sha256": "6cb584811b24e6267262be6b26c8c9236a61181e5b90766f92e2da104705dad5", + "rows": 4, + "bytes": 748 + }, + "tasks/regression/context.parquet": { + "sha256": "df3cf924ddc362b94ea6a7672ebb3e4ed52838a42bcf7b04e5520a91e939c19b", + "rows": 2, + "bytes": 1750 + }, + "tasks/regression/predict.parquet": { + "sha256": "2fcf0a7d39150dc136caf5ddf06c51156eb0e147dfa50c6914141eb78f3dc256", + "rows": 4, + "bytes": 1148 + }, + "tasks/regression/labels.parquet": { + "sha256": "204b2a6f889f3a92e9db52b54d54e14ff6ad043be2a05ab26f57e1a68329ef6a", + "rows": 4, + "bytes": 808 + } + }, + "writer": { + "pyarrow": "24.0.0", + "row_group_size": 3, + "reverse_layout": false + } +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/categories/part-00000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7436497b8d05c94f17c8a2dfc8c1e7462c4e599b GIT binary patch literal 755 zcmb7C%}(1u5FR_0YtbA6>aKPrA94UIhwu+=LE$itKp@f~PSOUbQbJ+Rj$KN-z>jP{q3rCd2Y=a!k0F-JUATw{`1qfG6RjFPg z;QGqTE6U4zB~;$YPEQj>!LtfBeOwXMT;o&2r9wOD^^!iLQGYayPv7c%hKie2l$-Uw z2n9VK!AtWun)m>CTHjV1qC!Hpf|+QXkQhv+5B;|nQgD0tVOmj$aZK>#C^muh8^5N>|niPvSEn$jZqzOk3Ge? zr-)&=J;@_}|4F1@Io!#TW5xzD$fUy2@zHWOkBQIJ>G;LEWV<{E*&&NftZtg>X4Dsa zLn?=VkN-5gt(A@I`1i)3WriB_yu_0&!A;qcTU(&YgDAZi#h3H#!C*d0(_~z}iqqkF Y(l4*9Ei5f8=_k;s6u|dR%K0or#%8oTcY_KQ_70g5j%{KlVO+41F>w$9YZ8AA8Ag^5Nx_zd ze8xb7ONtaODcmK002M=#GR#m=F%%RODJb#>(4|P;+r15U+9P&mvSaIZ-+T9d@5?J+ z7xgKcrIU1`Mjc8{5#nVnGHlJ-M6FEpq=dROUk0NQ_;re&t2Lf$MK+9_Ns*3@f;yYG zGh+luB|27v2a-l+)r%$a=U-|8

Fbh_>)tZG#X~Dj~xqjM}lROcoFY>eZ+%Nx3lx z6g%xkKiC*#oF``-rSYO?(wAdIP4=IjemD;v_kVc)^8)CNsI|s(wJk<%N|U6F4N;HK zf{xlPs=5{~rVym6a1jX6P7(}hk|1+Q5VGoZ%)z0#GSo}AS8F_1?~tJPAEc6yaV;C` zl<=)PrlbGhTWgYCHKnoWZA%f3=&EM?Yge_eM6|j)X0%#^-BcYP?52{YM4Kc;-AN%B zeLJ$7Q4ap*41u|Awu7w(%|Q#siT7!z8wUMGGwf`4Z+13NXnRAuJ3(Kb@_6cm!4{mx zNezcgTI7PeK{vb^cD91?LhYx~Vp&E-b71c@wzhY>Az`jx_HS@fq*;H4I3xZbjM~g$ zuPpZ3;&tx+w_Cz5kO&MC{wc>nIsr*|DG7%*e`^85ybiE@rW6`9$e8`i@j1cU z0a^JdDP11PI>75OBE|{EQv>-@E2J~LeJJQ?hFH%NFo;{ z5%o->ezn+d7JupRL%g|VU&*m9kB_6+(;?=Ue#SdKm`w;NWO9cC6 zN>lpv(V4KB%JIc8e>@Bf^D;q(DNSLhnb7l>G7~maGjVW&zs((YCh~G7wAb{R&`JD~ z=kHGp%)}8lPkN&?&YTI2`r{;jcJlw22}H5?4zr=h>=1^c=;cMder@@Jx}CiJ?B}@7 zwJJi0Mfk;aMTk97X^D9uTB6#LvI-bn#a&g11$@F;KjHK}iWbnrJ-L&vzQip$7V{4P zg2zj9m+%ZuV9BlAiB))1+Uv_-O|5rT7{CYrs~Cq`t?D9%iiXv;6y3&8>0cf(ZXwoI z);8C>^;;X4H^Vh>1t^Gn^`E%G1VoZChxSWL^*QnFcRId#8B}EtRk8S*aIEw;E-!^E z)0@CuM6T4s2PEIZIKRfa!IDbcfJs&PlSvHvn`%xk-IxAqKSq2}k&%IOD88!io~k>% ztO|o}RsP$!jpct0&yD4o$+eYl+NfsJ#l+;qq<;gxkK#Y%9bC@W;lI?s0r4{j`2YX_ literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/items/part-00000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..002ecc0daab0e0f9bb425ed5f24d5a43ff0f3749 GIT binary patch literal 1056 zcmb7E&2G~`5MIab)=F|nk#=QEMv72tD$$lU{SmEFq;&)o2qjomA-N#Ov657@F?K>! zjywg=z_BO9iDSiyH-R%JW_DwmRz-r5J+t$(-+VK&+z|d6GuRHhUuTq&8-(19DIp~7 z1xeJIfzx2vixLctPDW|eVT75ixCF%|c&W2R7OBcuGqQAU4r+~JPM;?LTxRoiFbG_# z&O!#oRE7W+W%&y%%gi#CnbmT#cuiC`(M?S^0DPbM8Ts2E3xKW2m;1x$I1jF*CB1(d zBY#nUX%@S_Ni3R%z34b#l(6Cgv1YNy>waf})E(%=42TuNU>J1=$tlEDRiP85VbYI> z{lUqkl2hBUt1fX^(cUDqWdDI;IZpW}jh|_5+cmG7tQ&vRbTC=`eekXr;~loHU3?5f9mcBeVYE|yQ3jzD=KwGdB*7z?i}9tzPFwXTeMo)FLQR1@McTJSRq z9;Rv!FuspULe-iGjh^SD79lFQmq{HDtw)FNo9j>FaL&zkoQD9c@vt#FpBS3%>iK9yno{dI6?%_qpk4e z6~%vdmssUENZyX3;c^(q%R!P1#^rZWlEODxu57HVuB_Ts_=eR(e&h&gz`x-)i@DJW literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tables/users/part-00000.parquet new file mode 100644 index 0000000000000000000000000000000000000000..2d39202d972d1a435954bdadb99fb17be6cf3c02 GIT binary patch literal 913 zcmb7DPj3=I6o0e8xQC5tH8Yt>_P{|lP19B&))^jos0J#FOopRl85JX7sEQjuD zF^cBvJda^3XA_EreGeqzCm6Z~NQ+9cLdD)O%8ny=767dYtjA{MJ5o54?~ zqShy+ebhm60OUVN*qC~c>Pszj z7U{_9Y1ryi+KQ!dIPRBA zc5OdiVMw=0N~!gToOuQ+KMiMBi)g+(m`-=Y+3a$ezm8_{=yH-TKiVtq6?Fx_x8#A( KIQjDG+t)W=Z( literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/context.parquet new file mode 100644 index 0000000000000000000000000000000000000000..0873540bc28cbf426b084b30771b3600916cfff1 GIT binary patch literal 1689 zcmb7F%}(1u5Z)xljUcK5!j>)hkVCOJKue*ipyIL)Bmt5ZERi1xDRL5%1e~8H1{zL% zfL5y1OCO_0jyQ4nuhzXA}u#h=I zqMO7W0e2OMzzfq$a2^G~D^l1yk;0y@NEXLfXJ_F&6b^)f!x|1ljQslJj7b_XB85Gl zdkw#JKqD&5XM`ZjBdm}g?>9asc{0lFok0Aq4|03>-s*941mb@v%z4}oq!1+5H+#d> z&&gs4%o0ummj?rjeY^UZ9vhf@0wK7F1vX;!bIVqoNPMqSH=Bkq4UsMoX{OQY*+#2k zmKs&ZGHtf(hTSinntj&=j_d7i$!;{wh55|8S!IbmrVxSqylr%AroBKi3K$vXoMn_v zt7M!T4M3JzGHm}fk&_kqNF|EE$#KF*<$)N4pvZ||0^)T*O{?Mm>mV6Bj|f40$0#g> z_*Hy)`9wYh)tkVbjK5tbHqpOjW_;CRI-djLRY1KCsh>iiGuk>68*02x#4VazcqY*m zFV)|idc!G0Z8c1>DMn;o=7%aX=D~cC-%hDtr!ex7-kR8P!?J{k-+7T!!{RMZWJQWl zU4NB-;dG+(mV1I@t)@{*58#VqkJ1aO)o7B^8co!gO#_E4q6DQoxWe9BVRwk8EHb=7 zl)ivTWBy1?wG$9oN3^$%bAm}Rh+C>5mDVffcdc@Aqh4($JLOceQBLKKiv{rA_3rPW zE)1mBa=Gm+E$jm|%F#ncnnrk+3V31sX}w-)R;?;PxDUGMn%KeA0nkJpXL^d0!;gq& za4P2GCq`CRmah3-CT)+6GU}D!24b zHEkUmtGPJPtb6e|I}SjXtTJ~Qp*N~g1Ibi3_~*tyy5HQKG%>a)iyk5}X)_D1(m63{r~^~ literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/labels.parquet new file mode 100644 index 0000000000000000000000000000000000000000..49d0f282695bcc721d137bb329cc84dc82d144cc GIT binary patch literal 748 zcmb7CU2D`(5S@Gw(;8tb-jFLn_NAc@+ct`;wY1R5V)0|G?V`532oiT=T)}LbY`X}3 z><{sm^`R7hgOASKWYLN*4rFHT%-nO%aI=rS`@$4EVxuc;0k;8~<_H!vNEUAFz$d6I zt)l=Ma$4jX!V;G6+4GKPkLnp0R{l|4TB5e;=*ATg-Cc37OM&Tbg$8xLC8bUwn-GH~ z7JoB5f)#_Bo}NtzFq=A?v82j*|KZUP-Me|CsHC{loT`!Mi8 literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/binary_classification/predict.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dcbf9b7edc0ffa95537e453080bf8670b3dc9299 GIT binary patch literal 1148 zcmb7EPj3=Y5T6Gdd~G>^>D%lkd*ITIF|<}%5^8$r1GJ)4+l_@bF(C_V0n0yuO7QH_ z597f^Pksnbo=l^S$9@41I`d##3}WJBcV^zq?C&=-FWG(R+!dPG5vjc30&W7tv??qx z@D^_DzzK|{Hn9S*C>%!>5g`$>4DM$PUY*qFaO*G8XcW_0I1s!5q+1t>JOU7%p)p`A z8-t&}#&UZu7gL!S7~HQVuG#1n*Bj!Bh~>NpC@yZn`R4Zzxe(wWA+b0;h@UaI?-97~ zUq=;qkTZ_hWSIsZ0l#nogwH!XA91`Cms5Q><8fQf+4^>0l7%IxPh|G(9naa;t50L>ji12D}p z)u1Bz%=9nJejl)ZFr=B?@7%5bn`UWhY{m1A=^vSW8MHqIk!N<*b1z_)N59Z?Zgc%o z1Jg3X(s7;^?~O0UI1k(*8OpSj@&I3w$5LXTCLc&ylbM=|rLka-yo{7NvUv6_o+y<$ zr1&#Z3M!K$XgX!;q~$3oBAQ5^>~S&q!rk2(9c?(>a`BMx5$yPi2TL%dTFX*>q2OfY z-fKio(J^zMbb3fyx#URM!8hG3x1DbFVe`aI6p~n)^81mhQv^*P5I9OjDsXJ>8YHJk zAE~^*8bl$g-R4N~BPnt)GCeBr(?{OcM2_Aif>q7w})^&R?&%D8ErQ2O~d%eyu e_R8z^8=ZD6vAMRswr*})0OJR?f&lyY-}w!f>Hd=d literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/context.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cc7046d099f56404d9f25245af374effcfbc48d4 GIT binary patch literal 1750 zcmb7F&2HL25MF~ztfU7L#g;Akz@f2vNKz67sZ{M@O=8Gj8&Crwii8X};MV*U98hjO z_K-@Edg){I$g#&rz2p)41bKlT+S#>@Hw9941-r8|v)?y=h$CFs5P2~nJ~@B`Eg=+t ziTuU+r-;OuIKK+^I0!LS76OYb1ilh_Iz_v<2==JJMI(Q6cnAsf>$kUNaU_r|1U?}L ztG&k&7ncvj2+c!K(a+E6FGdVaVh0w8-}fQdfgEZBY#!0%8SJ0}lh}bJtWp>^f@}i& zY;2VHF}ll5joY_CguX=Mp3t(`ijzWetY2$dEmNF_L{^ZzV7L3O*{)d?yY6Y`t+s2s zmzDR{rSAgfW~Wzi?UuE&d~mg>uF_u9_c7P(H7wWjTyo5+0E57R6#b)r1qIBCRbH&N|t6nG|QYUZRp%;T4wq4@YnyXD8Ju%i{G!fOAS%9KVj zbfcK|iTw71$qS{KGdP)%bUi2WnjnqlkfO?uaiQ?0^un7>`HFu=))v+=#(DUXwU6-- z>kVvR+`yRz)#>1HNTdwLyX1s>aKi1FsPY8il1QF~`V(NRzXg)#MB3jWdxla`kzBD( z9_amYY0s$|>1MrUbgNmzu4aqpNI-TeXqJfMQ~;I4 zpDQ$Lt-4bOu`u?B^kCWJdxyT?qgpCbDMPTc#zlFvIjA}5q*>Zrt4ThI~d2~#Rs5JT-t;j=?knp z1Cp6qPuvGLy<2U+PL@llcc;a4_mp;JNcsmj6pvu(VW#*Mk`rCC_u;~F*J|DFn%V1h fhKYgI>)V}nVsmRfxt>(cRD{?My5kW#f`7C>&a`DP literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/labels.parquet new file mode 100644 index 0000000000000000000000000000000000000000..21c27b95de7dfa6c7cd6749cc5f653ada6a2c154 GIT binary patch literal 808 zcmb7@%}(1u5XZ+KG7HTiAnnSQeDI;QI3y4vA`s%hIBlwwP#_Y7AS7fZ#zcq^oIoT_ zJPU_BM~{7q9;&L((OEkyr~(H@_RQ?gf9ALACkmg^zZ0GJy4j933X?b*fnbj7 z6nt>0u>w}!`A^Gr!NhM({L=h0pS|zaPC;Ij>p(6~0pKNN%1;45H1SjOC%XSvBgE|O z-n{cHLOdCQ`AkdBx2w#~8#|LyWPSmM{nq_hhk( zSeKF3D$+`(!Snk2XBxUGWz{dz(=f6$l?h^--k{Vz`PM%@A#CG8>4WqTG@Rhsym7{* zW_3Qq%XG+<^dwc*kV)98JaoU-?%L(!i?DSl&r9cyF&Jm_Qt^>$K?K_ygvsq)G%9um jgJPH@H)HQUN`}3gzE@sfDXo;8f(t+o)aiFPp#R7}dQFAa literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/tasks/regression/predict.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dcbf9b7edc0ffa95537e453080bf8670b3dc9299 GIT binary patch literal 1148 zcmb7EPj3=Y5T6Gdd~G>^>D%lkd*ITIF|<}%5^8$r1GJ)4+l_@bF(C_V0n0yuO7QH_ z597f^Pksnbo=l^S$9@41I`d##3}WJBcV^zq?C&=-FWG(R+!dPG5vjc30&W7tv??qx z@D^_DzzK|{Hn9S*C>%!>5g`$>4DM$PUY*qFaO*G8XcW_0I1s!5q+1t>JOU7%p)p`A z8-t&}#&UZu7gL!S7~HQVuG#1n*Bj!Bh~>NpC@yZn`R4Zzxe(wWA+b0;h@UaI?-97~ zUq=;qkTZ_hWSIsZ0l#nogwH!XA91`Cms5Q><8fQf+4^>0l7%IxPh|G(9naa;t50L>ji12D}p z)u1Bz%=9nJejl)ZFr=B?@7%5bn`UWhY{m1A=^vSW8MHqIk!N<*b1z_)N59Z?Zgc%o z1Jg3X(s7;^?~O0UI1k(*8OpSj@&I3w$5LXTCLc&ylbM=|rLka-yo{7NvUv6_o+y<$ zr1&#Z3M!K$XgX!;q~$3oBAQ5^>~S&q!rk2(9c?(>a`BMx5$yPi2TL%dTFX*>q2OfY z-fKio(J^zMbb3fyx#URM!8hG3x1DbFVe`aI6p~n)^81mhQv^*P5I9OjDsXJ>8YHJk zAE~^*8bl$g-R4N~BPnt)GCeBr(?{OcM2_Aif>q7w})^&R?&%D8ErQ2O~d%eyu e_R8z^8=ZD6vAMRswr*})0OJR?f&lyY-}w!f>Hd=d literal 0 HcmV?d00001 diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json new file mode 100644 index 000000000..69edb8031 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/bundle/validation.json @@ -0,0 +1,31 @@ +{ + "status": "passed", + "tables": { + "users": { + "rows": 3 + }, + "interactions": { + "rows": 4 + }, + "items": { + "rows": 3 + }, + "categories": { + "rows": 2 + } + }, + "relations": { + "interaction_user": { + "null_links": 0, + "edges": 4 + }, + "interaction_item": { + "null_links": 0, + "edges": 4 + }, + "item_category": { + "null_links": 0, + "edges": 3 + } + } +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json b/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json new file mode 100644 index 000000000..7cb94b0e1 --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/fixture/workload.json @@ -0,0 +1,34 @@ +{ + "version": 1, + "logical_id": "b1b964d2a964e4fb95b4cb093d5de467199f2b10b67970e8fe3b5407d7b8137a", + "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", + "task": "binary_classification", + "prediction_cutoff": "2026-02-01T00:00:00Z", + "selection": "native_multihop", + "deduplication": "per_instance", + "fanout_scope": "per_source_relation_hop", + "layers": [ + [ + { + "relation": "interaction_user", + "direction": "reverse", + "fanout": 1 + } + ] + ], + "projections": { + "users": [ + "user_id", + "segment", + "active" + ], + "interactions": [ + "interaction_id", + "user_id", + "item_id", + "event_time", + "amount" + ] + }, + "encoding": "arrays" +} diff --git a/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb index f7e464471..d85a84a66 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb +++ b/packages/experimental/kumo-fm/examples/local_sdm/walkthrough.ipynb @@ -7,7 +7,7 @@ "source": [ "# Local FM Arrow to SDM\n", "\n", - "Provide a validated bundle, workload, compatible DiskGraph index path, and pinned SDM environment described in this folder's README. This notebook samples once and reuses the same Arrow batch for local inference and both HTTP bodies.\n" + "Run from this directory to use the included validated fixture, or set `FM_SDM_FIXTURE` for your own bundle and workload. Install the matching DiskGraph and pinned SDM environment described in this folder's README. This notebook samples once and reuses the same Arrow batch for local inference and both HTTP bodies.\n" ] }, { @@ -36,7 +36,7 @@ "from adapter import arrow_to_sdm\n", "from sdm.models import KumoRelational\n", "\n", - "fixture = Path(os.environ[\"FM_SDM_FIXTURE\"])\n", + "fixture = Path(os.environ.get(\"FM_SDM_FIXTURE\", \"fixture\"))\n", "torch.set_num_threads(2)\n", "bundle_path = Path(os.environ.get(\"FM_SDM_BUNDLE\", str(fixture / \"bundle\")))\n", "with fm.prepare(bundle_path, fixture / \"workload.json\",\n", From 5f6e066db769b01d86f01d2df6511f18a5a5f149 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:03:45 +0000 Subject: [PATCH 04/12] Adopt engine snapshots for FM payload orchestration --- .github/workflows/kumo-fm.yml | 13 + packages/experimental/kumo-fm/README.md | 37 ++ .../kumo-fm/examples/local_sdm/README.md | 17 + .../local_sdm/connector_to_prediction.py | 72 ++++ .../kumo-fm/src/kumo/fm/__init__.py | 3 +- .../kumo-fm/src/kumo/fm/_pytorch.py | 9 +- .../kumo-fm/src/kumo/fm/_snapshot.py | 390 ++++++++++++++++++ .../kumo-fm/src/kumo/fm/loader.py | 30 +- .../kumo-fm/tests/test_connector_snapshot.py | 106 +++++ .../kumo-fm/tests/test_snapshot_api.py | 95 +++++ .../tests/test_snapshot_integration.py | 139 +++++++ 11 files changed, 900 insertions(+), 11 deletions(-) create mode 100644 packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py create mode 100644 packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py create mode 100644 packages/experimental/kumo-fm/tests/test_connector_snapshot.py create mode 100644 packages/experimental/kumo-fm/tests/test_snapshot_api.py create mode 100644 packages/experimental/kumo-fm/tests/test_snapshot_integration.py diff --git a/.github/workflows/kumo-fm.yml b/.github/workflows/kumo-fm.yml index 652de67bb..42d8e33a4 100644 --- a/.github/workflows/kumo-fm.yml +++ b/.github/workflows/kumo-fm.yml @@ -63,3 +63,16 @@ jobs: cd "$RUNNER_TEMP/stage" ../fresh/bin/python -I -c 'import kumo.fm, kumo.pytorch, diskgraph; print(kumo.fm.__file__)' ../fresh/bin/python -m pytest -q tests + - name: Build matching connector and engine wheels + run: | + for package in kumo-api kumo-pql kumo-settings kumo-core kumo-engine; do + uv build --wheel --package "$package" --out-dir "$RUNNER_TEMP/engine-wheels" + done + - name: Test connector snapshot to FM from installed wheels + working-directory: ${{ runner.temp }}/stage + run: | + uv pip install --python ../fresh/bin/python --torch-backend cpu \ + ../engine-wheels/*.whl + uv pip check --python ../fresh/bin/python + ../fresh/bin/python -I -c 'import kumo.engine, kumo.fm, kumo.pytorch, diskgraph' + ../fresh/bin/python -m pytest -q tests/test_connector_snapshot.py diff --git a/packages/experimental/kumo-fm/README.md b/packages/experimental/kumo-fm/README.md index 73e9c0e1c..d97b88cdc 100644 --- a/packages/experimental/kumo-fm/README.md +++ b/packages/experimental/kumo-fm/README.md @@ -1,5 +1,42 @@ # Kumo FM payload loader +The primary workflow starts with a graph built through `kumo.engine` connectors. +Adopt its snapshot with `api.DiskGraphEngine.from_snapshot(snapshot, +work_dir=...)`, declare a public PyTorch `SampleSpec` with one +`NeighborhoodSpec` and a bounded future `Aggregate` or `Compare` target, then +call `fm.prepare(engine=..., sample=..., target=..., task=..., features=...)`. +See the runnable +[`connector_to_prediction.py`](examples/local_sdm/connector_to_prediction.py). +FM derives table and relation metadata from `engine.schema`; it creates +context/query anchors, computes context labels in a separate public PyTorch +plan, samples one columnar graph plan over the same index, and aligns both by +occurrence ID. The base index remains unchanged. Columnar graph output does not +currently fuse computed labels into the graph batch. +For an advanced public relation alias or table projection, pass +`schema=api.DataSchema(...)`; PyTorch checks it against the adopted index. + +`fm.Context(entities=..., times=...)` produces entities × times, with time as +the outer axis. `fm.Query(entities=..., at=...)` uses one query time. Either +accepts ordered `pairs=[(entity, time), ...]`; a third element supplies an +explicit int64 occurrence ID. Repeated pairs remain separate. Without explicit +IDs each split uses zero-based positions. Context's future label window must be +fully observable before the earliest query time. Query targets never enter the +request. `loader.materialize(context=..., query=...).load()` and the shortcut +`loader.load(context=..., query=...)` return the same `PayloadBatch` type as +the frozen-bundle path. `batch.tables(split)` and both `batch.encode(...)` +methods reuse its retained Arrow tables without sampling again. + +The snapshot path currently accepts a single public neighborhood, bounded +`recent` fanouts, supported scalar FM wire types and `Aggregate(Rows(..., +time=TimeWindow(start=0, stop=positive)))` or `Compare` over that aggregate. +Multiple `GraphLayer`s in that neighborhood are passed to one native sampling +operation. Classification requires an explicit two-entry raw-label-to-name +`classes` map; regression accepts a numeric aggregate. Unsupported types, +timeless destinations and uniform sampling fail explicitly. Native labels and +graphs remain separate materializations over the adopted snapshot. + +## Frozen bundle compatibility + Experimental `kumo-fm` builds the existing Kumo Relational HTTP request from public `kumo.pytorch` columnar graph batches backed by DiskGraph. Sampling and FM assembly happen once per `PreparedRequest.load()`. The returned diff --git a/packages/experimental/kumo-fm/examples/local_sdm/README.md b/packages/experimental/kumo-fm/examples/local_sdm/README.md index 8d647b085..0299556b3 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/README.md +++ b/packages/experimental/kumo-fm/examples/local_sdm/README.md @@ -1,5 +1,22 @@ # Optional local SDM example +`connector_to_prediction.py` is the primary connector-to-model example. Run it +from this directory after installing matching `kumo-engine`, DiskGraph, +`kumo-pytorch` and `kumo-fm` wheels from one engine revision, plus the pinned +SDM checkout in `requirements.lock`. `kumo.engine.datasets.shop()` supplies +the reproducible 1,000-user shop Parquet sample; the script writes producer and +consumer state under `scratch/connector-to-prediction/` here. The public +`KumoRelational` classifier downloads its pretrained checkpoint on first use; +set `HF_HOME` to a writable model-cache directory. A CUDA device is used when +available, otherwise CPU. The example's 30-day future label window ends well +before the November query cutoff. Its one sampling hop is independent of the +model's two-hop setting. No SDM package is imported by `kumo.fm` itself. + +```bash +cd packages/experimental/kumo-fm/examples/local_sdm +python connector_to_prediction.py +``` + This example keeps SDM and its Arrow compatibility handling beside the notebook. Install matching DiskGraph, `kumo-pytorch`, and `kumo-fm` wheels from one engine revision, then the pinned SDM environment in diff --git a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py new file mode 100644 index 000000000..2c07fdd1b --- /dev/null +++ b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py @@ -0,0 +1,72 @@ +"""Connector → snapshot → retained FM Arrow batch → local SDM prediction. + +Run from this directory after installing the matching engine/native wheels and +the optional SDM environment. See README.md for the checkpoint and dataset. +""" +from datetime import datetime, timezone +from pathlib import Path + +import kumo.engine as kumo +from kumo.engine import datasets +from kumo.pytorch import api +from kumoapi.typing import Stype +import torch +from sdm.models import KumoRelational + +from kumo import fm +from adapter import arrow_to_sdm + + +root = Path("scratch/connector-to-prediction").resolve() +root.mkdir(parents=True, exist_ok=True) +kumo.configure(scratch_dir=root / "producer") +source = kumo.FileConnector(root_dir=datasets.shop()) +graph = kumo.Graph( + tables={ + "users": kumo.Table.from_source_table( + source["users.parquet"], primary_key="user_id"), + "orders": kumo.Table.from_source_table( + source["orders.parquet"], primary_key="order_id", + time_column="order_time").with_column("user_id", stype=Stype.ID), + }, + edges=[kumo.Edge(src_table="orders", fkey="user_id", dst_table="users")], +) +engine = api.DiskGraphEngine.from_snapshot( + graph.snapshot(), work_dir=root / "consumer", num_threads=2) +relation = engine.schema.relations[0].name + +horizon = 30 * 24 * 60 * 60 +sample = api.SampleSpec( + entity=api.EntitySpec("users"), + targets={"next_orders": api.Compare( + api.Aggregate(api.Rows(path=(relation,), time=api.TimeWindow( + "order_time", start=0, stop=horizon)), "count"), + "greater_than", 0)}, + neighborhoods={"history": api.NeighborhoodSpec(layers=( + api.GraphLayer(samples=(api.GraphSample( + relation=relation, direction="reverse", strategy="recent", fanout=8),)), + ))}, +) + +with fm.prepare(engine=engine, sample=sample, target="next_orders", + task="binary_classification", classes={False: "no", True: "yes"}, + features={"users": ["user_id"], + "orders": ["order_id", "order_time"]}) as loader: + batch = loader.load( + context=fm.Context(entities=range(32), times=[ + datetime(2025, 6, 1, tzinfo=timezone.utc)]), + query=fm.Query(entities=range(8), + at=datetime(2025, 11, 1, tzinfo=timezone.utc)), + ) + +device = "cuda" if torch.cuda.is_available() else "cpu" +inputs = {name: value.to(device) for name, value in arrow_to_sdm(batch).items()} +model = KumoRelational(task="classification", device=device).eval() +with torch.inference_mode(): + model.fit(inputs["x_context"], inputs["y_context"], + inputs["related_context_tables"], num_estimators=1, num_hops=2, + generator=torch.Generator(device=device).manual_seed(42)) + predictions = model.predict(inputs["x_query"], inputs["related_query_tables"]) +print(predictions.to_arrow()) +print("Same retained batch, HTTP bytes:", len(batch.encode("arrays")), + len(batch.encode("arrow_ipc"))) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py index 8aa52a40d..73c6ba1f1 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py @@ -1,5 +1,6 @@ """FM payload assembly through public Kumo PyTorch columnar batches.""" from .batch import PayloadBatch from .loader import FMLoader, PreparedRequest, prepare +from ._snapshot import Context, Query -__all__ = ["FMLoader", "PreparedRequest", "PayloadBatch", "prepare"] +__all__ = ["Context", "Query", "FMLoader", "PreparedRequest", "PayloadBatch", "prepare"] diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py index b5319cb82..d29f7b34a 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py @@ -243,10 +243,13 @@ def _adapt_columnar_batch(self, workload, seeds, batch): [table[c].combine_chunks() for c in table.column_names], schema=table.schema)] for rel_id, (name, edge) in enumerate(graph.edges.items()): rel = self.bundle.relations[name] - if (edge.source_type, edge.destination_type) != (rel["child"], rel["parent"]): + orientation = (edge.source_type, edge.destination_type) + if orientation not in ((rel["child"], rel["parent"]), + (rel["parent"], rel["child"])): raise ValueError("kumo-pytorch relation orientation differs from the schema") + native_types = orientation columns = [] - for column, table_name in (("source", rel["child"]), ("destination", rel["parent"])): + for column, table_name in zip(("source", "destination"), native_types, strict=True): values = edge.index[column] if table_name in inverse: indices = values.to_numpy(zero_copy_only=False) @@ -254,6 +257,8 @@ def _adapt_columnar_batch(self, workload, seeds, batch): raise ValueError("kumo-pytorch edge endpoint is out of bounds") values = pa.chunked_array([pa.array(inverse[table_name][indices], pa.uint64())]) columns.append(values.cast(pa.uint64())) + if orientation != (rel["child"], rel["parent"]): + columns.reverse() index = pa.Table.from_arrays(columns, names=["id_a", "id_b"]) directions[rel_id] = name, 'forward', rel["child"], rel["parent"] edges[rel_id] = index.to_batches() or [pa.RecordBatch.from_arrays( diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py new file mode 100644 index 000000000..ab0c8140f --- /dev/null +++ b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py @@ -0,0 +1,390 @@ +"""FM orchestration over an already indexed public PyTorch engine.""" +from __future__ import annotations + +from contextlib import ExitStack +from dataclasses import dataclass +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace +import tempfile +import uuid + +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from ._assembly import GraphAssembler +from ._pytorch import KumoPytorchBackend +from .batch import PayloadBatch +from .bundle import TYPES +from .workload import timestamp, validate_scalar + + +@dataclass(frozen=True) +class Context: + """Historical examples: entities × times, or ordered explicit pairs. + + A pair may include a third, explicit int64 occurrence ID. Repeated pairs + remain distinct; omitted IDs are their zero-based positions in the split. + """ + + entities: object = () + times: object = () + pairs: object = () + + +@dataclass(frozen=True) +class Query: + """Prediction examples: entities at one time, or ordered explicit pairs.""" + + entities: object = () + at: object = None + pairs: object = () + + +def _rows(population, split, key_dtype): + if split == "context" and not isinstance(population, Context): + raise TypeError("context requires fm.Context") + if split == "predict" and not isinstance(population, Query): + raise TypeError("query requires fm.Query") + pairs = list(population.pairs) + entities = list(population.entities) + if pairs and (entities or (split == "context" and population.times) or + (split == "predict" and population.at is not None)): + raise ValueError("Choose explicit pairs or entities with times") + if not pairs: + if split == "context": + pairs = [(entity, at) for at in population.times for entity in entities] + else: + pairs = [(entity, population.at) for entity in entities] + if not pairs: + raise ValueError(f"{split} needs at least one occurrence") + result = [] + seen = set() + for position, item in enumerate(pairs): + if len(item) not in (2, 3): + raise ValueError("Occurrences require (entity, time[, instance_id])") + entity, at = item[:2] + validate_scalar(entity, key_dtype) + instant = timestamp(at) + identifier = item[2] if len(item) == 3 else position + validate_scalar(identifier, "int64") + if identifier in seen: + raise ValueError("Duplicate occurrence ID within a split") + seen.add(identifier) + result.append({"instance_id": identifier, "entity_key": entity, + "anchor_time": instant}) + return result + + +def _schema_view(schema): + """Derive FM's supported scalar schema without touching the base index.""" + from kumo.pytorch.contract.schema import table_clock + + tables = {} + for table in schema.tables: + if table.primary_key is None: + raise ValueError(f"FM requires a primary key: {table.name}") + columns = {} + for name, dtype in table.columns.items(): + arrow = dtype.to_arrow() + if pa.types.is_timestamp(arrow) and arrow.unit != "ns": + physical = "timestamp[us]" + elif pa.types.is_date32(arrow): + physical = "date" + elif pa.types.is_float64(arrow): + physical = "float64" + elif pa.types.is_boolean(arrow): + physical = "bool" + elif pa.types.is_int64(arrow): + physical = "int64" + elif pa.types.is_string(arrow): + physical = "string" + else: + physical = str(arrow) + if physical not in TYPES: + raise ValueError(f"FM cannot encode {table.name}.{name}: {arrow}") + columns[name] = {"dtype": physical, + "nullable": name != table.primary_key} + if columns[table.primary_key]["dtype"] not in ("int64", "string"): + raise ValueError(f"FM needs an int64/string key: {table.name}") + tables[table.name] = {"primary_key": table.primary_key, + "time_column": table_clock(table), "columns": columns} + relations = {} + for relation in schema.relations: + left, right = tables[relation.left_table], tables[relation.right_table] + if relation.right_key == right["primary_key"]: + child, parent, fk = relation.left_table, relation.right_table, relation.left_key + elif relation.left_key == left["primary_key"]: + child, parent, fk = relation.right_table, relation.left_table, relation.right_key + else: + raise ValueError(f"FM requires one primary-key side: {relation.name}") + relations[relation.name] = {"child": child, "parent": parent, "foreign_key": fk} + return SimpleNamespace(tables=tables, relations=relations) + + +def _target_horizon(value): + from kumo.pytorch import api + + aggregate = value.left if isinstance(value, api.Compare) else value + if not isinstance(aggregate, api.Aggregate) or not isinstance(aggregate.source, api.Rows): + raise ValueError("FM targets require a bounded Rows aggregate or comparison") + window = aggregate.source.time + if window is None or window.start != 0 or window.stop is None or window.stop <= 0: + raise ValueError("FM target needs a future window (0, positive stop]") + return window.stop + + +def _selected_graph(schema, sample, features, bundle): + from kumo.pytorch import api + + if len(sample.neighborhoods) != 1: + raise ValueError("FM requires exactly one declared NeighborhoodSpec") + graph_name, neighborhood = next(iter(sample.neighborhoods.items())) + frontier = {sample.entity.table} + tables, relations = set(frontier), set() + if not neighborhood.layers: + raise ValueError("FM requires at least one graph layer") + for layer in neighborhood.layers: + if not layer.samples: + raise ValueError("FM cannot sample an empty graph layer") + following = set() + for step in layer.samples: + if step.strategy != "recent" or not step.fanout or step.fanout < 0: + raise ValueError("FM supports bounded recent graph sampling only") + if step.relation not in bundle.relations: + raise ValueError(f"Unknown relation: {step.relation}") + rel = bundle.relations[step.relation] + source, dest = ((rel["child"], rel["parent"]) if step.direction == "forward" + else (rel["parent"], rel["child"])) + if source not in frontier: + raise ValueError(f"Unreachable graph layer: {source}") + if bundle.tables[dest]["time_column"] is None: + raise ValueError(f"FM cannot sample uniformly into timeless table {dest}") + following.add(dest) + tables.update((source, dest)) + relations.add(step.relation) + frontier = following + if set(features) != tables: + raise ValueError(f"features must name reachable tables: {sorted(tables)}") + projections = {} + fields = {} + for name in sorted(tables): + declared = bundle.tables[name]["columns"] + requested = list(features[name]) + if len(requested) != len(set(requested)) or not set(requested) <= set(declared): + raise ValueError(f"Invalid FM feature projection: {name}") + pk = bundle.tables[name]["primary_key"] + projections[name] = list(dict.fromkeys((pk, *requested))) + internal = list(projections[name]) + clock = bundle.tables[name]["time_column"] + if clock: + internal.append(clock) + internal.extend(bundle.relations[r]["foreign_key"] for r in relations + if bundle.relations[r]["child"] == name) + fields[name] = api.Fields(tuple(api.Encode(column, api.Raw()) + for column in dict.fromkeys(internal))) + return graph_name, fields, tables, relations, projections + + +class SnapshotFMLoader: + """Prepare FM labels and columnar graph plans over one adopted snapshot.""" + + def __init__(self, *, engine, sample, target, task, features, classes=None, + batch_size=1024, budgets=None, schema=None): + from kumo.pytorch import api + from .loader import PreparedRequest + + if not isinstance(engine, api.DiskGraphEngine) or not isinstance(sample, api.SampleSpec): + raise TypeError("Provide an adopted DiskGraphEngine and SampleSpec") + schema = engine.schema if schema is None else schema + if not isinstance(schema, api.DataSchema): + raise TypeError("schema must be a public DataSchema projection or alias") + if target not in sample.targets or task not in ("regression", "classification", "binary_classification"): + raise ValueError("Choose a declared target and supported FM task") + self._prepared_request_type = PreparedRequest + self._closed = False + self._engine, self._target = engine, target + self._bundle = _schema_view(schema) + self._entity = sample.entity.table + self._key_type = self._bundle.tables[self._entity]["columns"][ + self._bundle.tables[self._entity]["primary_key"]]["dtype"] + self._horizon = _target_horizon(sample.targets[target]) + self._kind = "regression" if task == "regression" else "binary_classification" + if self._kind == "regression": + if classes is not None: + raise ValueError("Regression has no classes") + self._classes = None + target_spec = {"dtype": "float64"} + else: + if not isinstance(classes, dict) or len(classes) != 2 or len(set(classes.values())) != 2: + raise ValueError("Binary classification requires a two-entry raw-value to class-name map") + self._classes = dict(classes) + target_spec = {"dtype": "string", "classes": list(classes.values())} + graph_name, fields, tables, relations, projections = _selected_graph( + schema, sample, features, self._bundle) + if not isinstance(batch_size, int) or batch_size < 1: + raise ValueError("batch_size must be positive") + default_budgets = {"max_instances": 10000, "max_rows_per_table": 1000000, + "max_payload_bytes": 30 * 1024**2, + "max_decoded_arrow_bytes": 30 * 1024**2, + "max_columns": 500, "max_candidate_rows": 1000000} + default_budgets.update(budgets or {}) + if any(type(value) is not int or value < 1 for value in default_budgets.values()): + raise ValueError("FM budgets must be positive integers") + self._batch_size = min(batch_size, default_budgets["max_instances"]) + self.workload = SimpleNamespace(bundle=self._bundle, entity=self._entity, + kind=self._kind, target=target_spec, table_names=tables, + relation_names=relations, projections=projections, budgets=default_budgets) + self._adapter = SimpleNamespace(bundle=self._bundle, + node_types=list(self._bundle.tables), assembler=GraphAssembler(self._bundle), + metrics={"native_candidate_rows": 0}) + self._stack = ExitStack() + self._temporary = tempfile.TemporaryDirectory(prefix="kumo-fm-anchors-") + pk = self._bundle.tables[self._entity]["primary_key"] + label_sample = api.SampleSpec(entity=sample.entity, + entity_values={pk: api.Field(pk)}, targets={target: sample.targets[target]}) + label_batch = api.BatchSpec(inputs={"entity": api.Input( + api.EntitySource(), api.Fields((api.Encode(pk, api.Raw()),)), api.Separate())}, + targets={"label": api.LabelTarget(target, api.Raw())}, + batching=api.Batching(max_samples=self._batch_size)) + graph_sample = api.SampleSpec(entity=sample.entity, + neighborhoods={graph_name: sample.neighborhoods[graph_name]}) + graph_batch = api.BatchSpec(inputs={"graph": api.Input( + api.NeighborhoodSource(graph_name), api.Graph(node_fields=fields), None)}, + batching=api.Batching(max_samples=self._batch_size)) + try: + self._labels = self._stack.enter_context(api.prepare( + contract=api.DataLoaderSpec(schema, label_sample, label_batch), engine=engine)) + self._graphs = self._stack.enter_context(api.prepare( + contract=api.DataLoaderSpec(schema, graph_sample, graph_batch), engine=engine)) + except BaseException: + self.close() + raise + + @property + def preparation_mode(self): + return "existing_snapshot" + + def __enter__(self): + return self + + def __exit__(self, *_): + self.close() + + def close(self): + if not self._closed: + self._closed = True + self._stack.close() + self._temporary.cleanup() + + def materialize(self, *, context=None, query=None): + if self._closed: + raise RuntimeError("Loader is closed") + if context is None and query is None: + raise ValueError("Provide context or query occurrences") + splits = {} + for split, population in (("context", context), ("predict", query)): + if population is not None: + seeds = _rows(population, split, self._key_type) + if len(seeds) > self.workload.budgets["max_instances"]: + raise ValueError("FM instance budget exceeded") + splits[split] = seeds + if "context" in splits: + if "predict" not in splits: + raise ValueError("Context labels require a query cutoff") + cutoff = min(row["anchor_time"] for row in splits["predict"]) + if any(row["anchor_time"] + timedelta(seconds=self._horizon) > cutoff + for row in splits["context"]): + raise ValueError("Context label window is not observable before the earliest query") + return self._prepared_request_type(self, splits) + + def load(self, *, context=None, query=None): + with self.materialize(context=context, query=query) as request: + return request.load() + + def _anchor(self, seeds): + from kumo.pytorch import api + + path = Path(self._temporary.name) / f"{uuid.uuid4().hex}.parquet" + pq.write_table(pa.table({ + "entity": pa.array([s["entity_key"] for s in seeds], TYPES[self._key_type]), + "sample": pa.array([s["instance_id"] for s in seeds], pa.int64()), + "at": pa.array([s["anchor_time"] for s in seeds], pa.timestamp("us", tz="UTC")), + }), path) + return api.AnchorTable(api.Parquet(path=str(path)), entity_id="entity", + sample_id="sample", as_of_time="at") + + def _labels_for(self, anchor, seeds): + result = {} + expected = {s["instance_id"] for s in seeds} + with self._labels.materialize(anchor) as task: + for batch in task.batches(output_format="inspectable", shuffle=False, + batch_size=self._batch_size): + label = batch.targets["label"] + sample_ids = list(batch.metadata["sample_ids"]) + values, valid = label["values"].tolist(), label["valid"].tolist() + if len(sample_ids) != len(values) or len(values) != len(valid): + raise ValueError("FM label batch is misaligned") + for identifier, value, present in zip(sample_ids, values, valid, strict=True): + identifier = int(identifier) + if identifier not in expected or identifier in result or not present: + raise ValueError("Missing, duplicated, or unexpected FM context label") + if self._classes is not None: + if value not in self._classes: + raise ValueError("Context label is outside declared classes") + value = self._classes[value] + else: + value = float(value) + result[identifier] = value + if set(result) != expected: + raise ValueError("Missing FM context labels") + return result + + def _graphs_for(self, anchor, seeds): + from ._contract import schema_for + + expected = {s["instance_id"]: s for s in seeds} + seen = set() + parts = {name: [] for name in schema_for(self.workload)["related_tables"]} + with self._graphs.materialize(anchor) as task: + for batch in task.batches(output_format="columnar", shuffle=False, + batch_size=self._batch_size): + identifiers = [int(value) for value in batch.metadata["sample_ids"]] + if (len(identifiers) != len(set(identifiers)) or + any(value not in expected or value in seen for value in identifiers)): + raise ValueError("Duplicate or unexpected FM graph occurrence") + seen.update(identifiers) + chunk = [expected[value] for value in identifiers] + local = SimpleNamespace(inputs=batch.inputs, + metadata={"sample_ids": list(range(len(chunk)))}) + related = KumoPytorchBackend._adapt_columnar_batch( + self._adapter, self.workload, chunk, local) + for name, table in related.items(): + parts[name].append(table) + if seen != set(expected): + raise ValueError("Missing FM graph occurrences") + order = {seed["instance_id"]: i for i, seed in enumerate(seeds)} + result = {} + for name, chunks in parts.items(): + table = pa.concat_tables(chunks) if chunks else pa.table({}) + if len(table): + rank = pa.array([order[int(value)] for value in table["instance_id"].to_pylist()]) + index = pc.sort_indices(pa.table({"rank": rank, + "key": table[self._bundle.tables[name]["primary_key"]]}), + sort_keys=[("rank", "ascending"), ("key", "ascending")]) + table = table.take(index) + result[name] = table + return result + + def _load(self, splits): + if self._closed: + raise RuntimeError("Loader is closed") + ready = {} + for split, seeds in splits.items(): + anchor = self._anchor(seeds) + if split == "context": + labels = self._labels_for(anchor, seeds) + seeds = [dict(seed, target=labels[seed["instance_id"]]) for seed in seeds] + ready[split] = seeds, self._graphs_for(anchor, seeds) + return PayloadBatch(self.workload, ready) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/loader.py b/packages/experimental/kumo-fm/src/kumo/fm/loader.py index 42c072809..e8fc0d089 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/loader.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/loader.py @@ -43,12 +43,7 @@ def close(self): def load(self): if self._closed or self._loader._closed: raise RuntimeError("Prepared request or loader is closed") - workload = self._loader.workload - ready = { - split: (seeds, self._loader._backend.extract(workload, seeds)) - for split, seeds in self._splits.items() - } - return PayloadBatch(workload, ready) + return self._loader._load(self._splits) class FMLoader: @@ -101,6 +96,25 @@ def materialize(self, *, context=None, predict=None): raise ValueError("Request needs at least one split") return PreparedRequest(self, splits) - -def prepare(dataset, spec, *, index, threads=2, verify_files=True): + def _load(self, splits): + ready = {split: (seeds, self._backend.extract(self.workload, seeds)) + for split, seeds in splits.items()} + return PayloadBatch(self.workload, ready) + + +def prepare(dataset=None, spec=None, *, index=None, threads=2, verify_files=True, + engine=None, sample=None, target=None, task=None, features=None, + classes=None, batch_size=1024, budgets=None, schema=None): + """Open a frozen bundle, or prepare FM over an adopted snapshot engine.""" + if engine is not None: + from ._snapshot import SnapshotFMLoader + if dataset is not None or spec is not None or index is not None: + raise ValueError("Snapshot and bundle inputs cannot be mixed") + return SnapshotFMLoader(engine=engine, sample=sample, target=target, + task=task, features=features, classes=classes, batch_size=batch_size, + budgets=budgets, schema=schema) + if schema is not None: + raise ValueError("schema is only valid with an adopted engine") + if dataset is None or spec is None or index is None: + raise ValueError("Bundle preparation needs dataset, spec, and index") return FMLoader(dataset, spec, index=index, threads=threads, verify_files=verify_files) diff --git a/packages/experimental/kumo-fm/tests/test_connector_snapshot.py b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py new file mode 100644 index 000000000..6416d6279 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py @@ -0,0 +1,106 @@ +"""Installed-wheel connector → snapshot → FM, including exported reuse.""" +import base64 +import hashlib +import json +import shutil +from datetime import datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from kumo import fm +from kumo.pytorch import api + +kumo = pytest.importorskip("kumo.engine") +from kumo.engine._scratch import get_scratch_dir # noqa: E402 +from kumoapi.typing import Stype # noqa: E402 + + +def inventory(root): + return {str(path.relative_to(root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in root.rglob("*") if path.is_file()} + + +def at(second): + return datetime.fromtimestamp(second, tz=timezone.utc) + + +def test_connector_export_reuse_and_aliased_relation(tmp_path): + source = tmp_path / "source" + source.mkdir() + pq.write_table(pa.table({"id": [-5, 9007199254740993]}), source / "users.parquet") + pq.write_table(pa.table({"id": [1, 2, 3, 4], + "user": [-5, -5, -5, 9007199254740993], + "at": pa.array([10, 20, 30, 10], pa.timestamp("s"))}), + source / "events.parquet") + connector = kumo.FileConnector(root_dir=str(source)) + graph = kumo.Graph(tables={ + "users": kumo.Table.from_source_table(connector["users.parquet"], primary_key="id"), + "events": kumo.Table.from_source_table(connector["events.parquet"], + primary_key="id", time_column="at").with_column("user", stype=Stype.ID), + }, edges=[kumo.Edge(src_table="events", fkey="user", dst_table="users")]) + previous = get_scratch_dir() + try: + kumo.configure(scratch_dir=tmp_path / "producer") + snapshot = graph.snapshot() + original_hashes = inventory(snapshot.index_path) + def run(snapshot, work): + engine = api.DiskGraphEngine.from_snapshot(snapshot, work_dir=work, + num_threads=2) + physical = engine.schema.relations[0] + alias = api.RelationSpec("events_selected", physical.left_table, + physical.left_key, physical.right_table, physical.right_key) + schema = api.DataSchema(engine.schema.tables, (alias,)) + sample = api.SampleSpec(entity=api.EntitySpec("users"), + targets={"next": api.Aggregate(api.Rows(path=(alias.name,), + time=api.TimeWindow("at", start=0, stop=10)), "count")}, + neighborhoods={"history": api.NeighborhoodSpec((api.GraphLayer(( + api.GraphSample(alias.name, direction="reverse", fanout=2, + strategy="recent"),)),))}) + with fm.prepare(engine=engine, schema=schema, sample=sample, target="next", + task="regression", features={"users": [], "events": ["at"]}, + batch_size=1) as loader: + batch = loader.load(context=fm.Context(pairs=[(-5, at(15), 88), + (-5, at(15), 89), (9007199254740993, at(15), 90)]), + query=fm.Query(pairs=[(-5, at(40), 91), + (9007199254740993, at(40), 92)])) + context = batch.tables("context") + assert [(row["instance_id"], row["__fm_target"]) + for row in context["instance_table"].to_pylist()] == [ + (88, 1.0), (89, 1.0), (90, 0.0)] + rows = context["related_tables"]["events"].to_pylist() + assert [(row["instance_id"], row["id"], row["__fm_edge_events_selected"]) + for row in rows] == [(88, 1, -5), (89, 1, -5), + (90, 4, 9007199254740993)] + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + arrays, ipc = batch.encode("arrays"), batch.encode("arrow_ipc") + assert arrays == batch.encode("arrays") and ipc == batch.encode("arrow_ipc") + wire_arrays, wire_ipc = json.loads(arrays), json.loads(ipc) + assert wire_arrays["context"]["related_tables"]["events"]["rows"][2][0] == 90 + assert str(9007199254740993) in arrays.decode() + for split in batch.splits: + direct = batch.tables(split) + entries = {"instance_table": direct["instance_table"], + **direct["related_tables"]} + for name, table in entries.items(): + encoded = (wire_ipc[split]["instance_table"] if name == "instance_table" + else wire_ipc[split]["related_tables"][name]) + decoded = pa.ipc.open_stream(base64.b64decode( + encoded["payload_base64"])).read_all() + assert decoded.equals(table) + return batch.tables("predict")["related_tables"]["events"] + first = run(snapshot, tmp_path / "work1") + assert inventory(snapshot.index_path) == original_hashes + exports = kumo.FileConnector(root_dir=str(tmp_path / "exports")) + location = snapshot.export(kumo.SnapshotOutputConfig(exports, "saved")) + shutil.rmtree(source) + shutil.rmtree(tmp_path / "producer") + kumo.configure(scratch_dir=tmp_path / "consumer") + reopened = kumo.GraphSnapshot.load_from_connector(exports, path=location) + exported_hashes = inventory(reopened.index_path) + second = run(reopened, tmp_path / "work2") + assert first.equals(second) + assert inventory(reopened.index_path) == exported_hashes + finally: + kumo.configure(scratch_dir=previous) diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_api.py b/packages/experimental/kumo-fm/tests/test_snapshot_api.py new file mode 100644 index 000000000..c272f2411 --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_snapshot_api.py @@ -0,0 +1,95 @@ +"""Snapshot input validation before any native graph is sampled.""" +from datetime import datetime, timezone +from types import SimpleNamespace + +import pytest + +from kumo.pytorch import api +from kumo.fm import Context, Query +from kumo.fm._snapshot import (SnapshotFMLoader, _rows, _schema_view, + _selected_graph, _target_horizon) + + +def schema(): + return api.DataSchema( + tables=( + api.TableSpec("users", "user_id", {"user_id": api.DType("int64")}, + time_column=api.NON_TEMPORAL), + api.TableSpec("orders", "order_id", { + "order_id": api.DType("int64"), "user_id": api.DType("int64"), + "at": api.DType("timestamp", unit="s", timezone="UTC"), + "amount": api.DType("float64")}, time_column="at"), + ), + relations=(api.RelationSpec("alias", "users", "user_id", "orders", "user_id"),), + ) + + +def test_population_order_and_repeated_occurrences(): + context = _rows(Context(entities=[7, 8], times=["2025-01-01T00:00:00Z", + "2025-02-01T00:00:00Z"]), + "context", "int64") + assert [(row["instance_id"], row["entity_key"]) for row in context] == [ + (0, 7), (1, 8), (2, 7), (3, 8)] + query = _rows(Query(pairs=[(7, "2025-03-01T00:00:00Z", 9007199254740993), + (7, "2025-03-01T00:00:00Z", -1)]), "predict", "int64") + assert [row["instance_id"] for row in query] == [9007199254740993, -1] + assert query[0]["anchor_time"] == datetime(2025, 3, 1, tzinfo=timezone.utc) + with pytest.raises(ValueError, match="Duplicate occurrence"): + _rows(Query(pairs=[(7, "2025-03-01T00:00:00Z", 1), + (8, "2025-03-01T00:00:00Z", 1)]), "predict", "int64") + + +def test_schema_alias_projection_and_target_window(): + bundle = _schema_view(schema()) + assert bundle.relations["alias"] == {"child": "orders", "parent": "users", + "foreign_key": "user_id"} + target = api.Aggregate(api.Rows(path=("alias",), + time=api.TimeWindow("at", start=0, stop=30)), "count") + assert _target_horizon(target) == 30 + sample = api.SampleSpec(entity=api.EntitySpec("users"), + targets={"next": target}, neighborhoods={"history": api.NeighborhoodSpec(( + api.GraphLayer((api.GraphSample("alias", direction="reverse", + fanout=2, strategy="recent"),)),))}) + name, fields, tables, relations, projections = _selected_graph( + schema(), sample, {"users": [], "orders": ["amount"]}, bundle) + assert (name, tables, relations) == ("history", {"users", "orders"}, {"alias"}) + assert projections == {"orders": ["order_id", "amount"], "users": ["user_id"]} + assert {item.field for item in fields["orders"].fields} == { + "order_id", "amount", "at", "user_id"} + with pytest.raises(ValueError, match="uniformly into timeless"): + _selected_graph(schema(), api.SampleSpec(entity=api.EntitySpec("orders"), + neighborhoods={"history": api.NeighborhoodSpec((api.GraphLayer(( + api.GraphSample("alias", direction="forward", fanout=2, + strategy="recent"),)),))}), + {"orders": [], "users": []}, bundle) + + +def test_context_labels_join_by_occurrence_id_across_batches(): + class Task: + def __init__(self, values): + self.values = values + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def batches(self, **_): + for identifier, value in self.values: + yield SimpleNamespace(metadata={"sample_ids": [identifier]}, + targets={"label": {"values": SimpleNamespace(tolist=lambda: [value]), + "valid": SimpleNamespace(tolist=lambda: [True])}}) + + loader = object.__new__(SnapshotFMLoader) + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3), (11, 9)])) + loader._batch_size = 1 + loader._classes = None + assert loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) == { + 22: 3.0, 11: 9.0} + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3)])) + with pytest.raises(ValueError, match="Missing FM context labels"): + loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) + loader._labels = SimpleNamespace(materialize=lambda _: Task([(22, 3), (22, 9)])) + with pytest.raises(ValueError, match="duplicated"): + loader._labels_for(None, [{"instance_id": 11}, {"instance_id": 22}]) diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_integration.py b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py new file mode 100644 index 000000000..c79294c2d --- /dev/null +++ b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py @@ -0,0 +1,139 @@ +"""Integrated multi-layer FM sampling over a reusable native index.""" +from datetime import datetime, timezone + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from kumo import fm +from kumo.pytorch import api + + +UTC = timezone.utc +LARGE = 9007199254740993 + + +def when(day): + return datetime(2025, 1, day, tzinfo=UTC) + + +def build_index(tmp_path): + rows = { + "users": [{"user_id": LARGE}, {"user_id": -5}], + "orders": [ + {"order_id": 11, "user_id": LARGE, "at": when(1)}, + {"order_id": 12, "user_id": LARGE, "at": when(2)}, + {"order_id": 13, "user_id": -5, "at": when(2)}, + {"order_id": 14, "user_id": LARGE, "at": when(15)}, + ], + "payments": [ + {"payment_id": 21, "order_id": 11, "at": when(3)}, + {"payment_id": 22, "order_id": 12, "at": when(3)}, + {"payment_id": 23, "order_id": 13, "at": when(3)}, + ], + "audits": [ + {"audit_id": 31, "payment_id": 21, "at": when(4)}, + {"audit_id": 32, "payment_id": 22, "at": when(4)}, + {"audit_id": 33, "payment_id": 23, "at": when(4)}, + ], + "notes": [ + {"note_id": 41, "audit_id": 31, "at": when(5)}, + {"note_id": 42, "audit_id": 32, "at": when(5)}, + {"note_id": 43, "audit_id": 33, "at": when(5)}, + ], + } + keys = {"users": "user_id", "orders": "order_id", "payments": "payment_id", + "audits": "audit_id", "notes": "note_id"} + sources, tables = {}, [] + for name, values in rows.items(): + path = tmp_path / f"{name}.parquet" + table = pa.Table.from_pylist(values) + pq.write_table(table, path) + sources[name] = api.Parquet(str(path)) + tables.append(api.TableSpec(name, keys[name], + {field.name: api.DType.from_arrow(field.type) for field in table.schema}, + time_column="at" if name != "users" else api.NON_TEMPORAL)) + relations = ( + api.RelationSpec("order_user", "orders", "user_id", "users", "user_id"), + api.RelationSpec("payment_order", "payments", "order_id", "orders", "order_id"), + api.RelationSpec("audit_payment", "audits", "payment_id", "payments", "payment_id"), + api.RelationSpec("note_audit", "notes", "audit_id", "audits", "audit_id"), + ) + schema = api.DataSchema(tuple(tables), relations) + graph = api.SampleSpec(entity=api.EntitySpec("users"), + neighborhoods={"history": api.NeighborhoodSpec((api.GraphLayer(( + api.GraphSample("order_user", direction="reverse", fanout=2, + strategy="recent"),)),))}) + contract = api.DataLoaderSpec(schema, graph, api.BatchSpec(inputs={ + "graph": api.Input(api.NeighborhoodSource("history"), api.Graph(node_fields={ + name: api.Fields(tuple(api.Encode(column, api.Raw()) for column in keys_and_types)) + for name, keys_and_types in ((table.name, table.columns) for table in tables + if table.name in ("users", "orders")) + }), None)}, batching=api.Batching(max_samples=2))) + index = tmp_path / "index" + with api.prepare(contract=contract, dataset=api.Dataset(sources), + engine=api.DiskGraphEngine(index=index, num_threads=2)): + pass + return index, schema + + +@pytest.mark.parametrize("depth", [2, 3, 4]) +def test_snapshot_multihop_and_occurrence_alignment(tmp_path, depth): + index, schema = build_index(tmp_path) + work = tmp_path / "consumer" + engine = api.DiskGraphEngine.from_index(index, work_dir=work, num_threads=2) + assert engine.schema == schema + names = ("order_user", "payment_order", "audit_payment", "note_audit") + sample = api.SampleSpec(entity=api.EntitySpec("users"), + targets={"future": api.Aggregate(api.Rows( + path=(names[0],), time=api.TimeWindow("at", start=0, stop=7*86400)), + "count")}, + neighborhoods={"history": api.NeighborhoodSpec(tuple( + api.GraphLayer((api.GraphSample(name, direction="reverse", fanout=2, + strategy="recent"),)) + for name in names[:depth]))}) + tables = ("users", "orders", "payments", "audits", "notes")[:depth+1] + features = {name: [] for name in tables} + context = fm.Context(pairs=[(LARGE, when(10), 100), + (LARGE, when(10), 101), (-5, when(10), 102)]) + query = fm.Query(pairs=[(LARGE, when(25), 200), (-5, when(25), 201)]) + with fm.prepare(engine=engine, sample=sample, target="future", task="regression", + features=features, batch_size=2) as loader: + batch = loader.load(context=context, query=query) + instance = batch.tables("context")["instance_table"].to_pylist() + assert [(row["instance_id"], row["__fm_target"]) for row in instance] == [ + (100, 1.0), (101, 1.0), (102, 0.0)] + assert "__fm_target" not in batch.tables("predict")["instance_table"].column_names + related = batch.tables("context")["related_tables"] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 100] == [11, 12] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 101] == [11, 12] + assert [row["order_id"] for row in related["orders"].to_pylist() + if row["instance_id"] == 102] == [13] + expected_nodes = { + "payments": ("payment_id", [(100, 21), (100, 22), (101, 21), + (101, 22), (102, 23)]), + "audits": ("audit_id", [(100, 31), (100, 32), (101, 31), + (101, 32), (102, 33)]), + "notes": ("note_id", [(100, 41), (100, 42), (101, 41), + (101, 42), (102, 43)]), + } + for name, (key, expected) in expected_nodes.items(): + if name in related: + assert [(row["instance_id"], row[key]) for row in + related[name].to_pylist()] == expected + expected_edges = { + "order_user": [LARGE, LARGE, LARGE, LARGE, -5], + "payment_order": [11, 12, 11, 12, 13], + "audit_payment": [21, 22, 21, 22, 23], + "note_audit": [31, 32, 31, 32, 33], + } + for step in names[:depth]: + child = {"order_user": "orders", "payment_order": "payments", + "audit_payment": "audits", "note_audit": "notes"}[step] + assert "__fm_edge_" + step in related[child].column_names + assert related[child]["__fm_edge_" + step].null_count == 0 + assert related[child]["__fm_edge_" + step].to_pylist() == expected_edges[step] + assert batch.encode("arrays") == batch.encode("arrays") + assert batch.encode("arrow_ipc") == batch.encode("arrow_ipc") From 8d9e857f1bc020974787bd139f11444c481a88ba Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:06:23 +0000 Subject: [PATCH 05/12] Clarify notebook fixture configuration --- .../experimental/kumo-fm/examples/local_sdm/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/experimental/kumo-fm/examples/local_sdm/README.md b/packages/experimental/kumo-fm/examples/local_sdm/README.md index 0299556b3..246fd8767 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/README.md +++ b/packages/experimental/kumo-fm/examples/local_sdm/README.md @@ -17,15 +17,16 @@ cd packages/experimental/kumo-fm/examples/local_sdm python connector_to_prediction.py ``` -This example keeps SDM and its Arrow compatibility handling beside the -notebook. Install matching DiskGraph, `kumo-pytorch`, and `kumo-fm` wheels +The separate `walkthrough.ipynb` uses the included frozen bundle fixture and +keeps SDM and its Arrow compatibility handling beside the notebook. Install +matching DiskGraph, `kumo-pytorch`, and `kumo-fm` wheels from one engine revision, then the pinned SDM environment in `requirements.lock`. The model may download the `nvidia/Kumo-Relational` v1.0.1 checkpoint; the prior validation used snapshot `2bd603d3d8f25f67a7aaa8579908595e20567e22`. -The example includes `fixture/bundle/` and `fixture/workload.json`. Run the -notebook from this directory to use them by default. Set `FM_SDM_FIXTURE` to +The notebook includes `fixture/bundle/` and `fixture/workload.json`. Run it +from this directory to use them by default. Set `FM_SDM_FIXTURE` to another directory containing `bundle/` and `workload.json` to use your own eligible `native_multihop` workload. Set `FM_SDM_BUNDLE` when its bundle has another path. The notebook opens or creates `fm-index/` in the fixture directory, samples one From eab33121d435dff32cd70466da771dab7d703996 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:08:54 +0000 Subject: [PATCH 06/12] Refresh frozen payload comparison on stacked wheels --- benchmark/fm-payloads/README.md | 4 +- .../reports/omnistation-smoke.json | 71 ++++++++++--------- 2 files changed, 38 insertions(+), 37 deletions(-) diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md index d0dfc1502..61f1bc7ec 100644 --- a/benchmark/fm-payloads/README.md +++ b/benchmark/fm-payloads/README.md @@ -37,8 +37,8 @@ counts. Median phase times in milliseconds: | Encoding | DuckDB reference full payload | DiskGraph load and assembly | Encoding retained batch | | --- | ---: | ---: | ---: | -| Arrays | 40.691 | 3.643 | 0.731 | -| Arrow IPC | 40.765 | 3.457 | 0.446 | +| Arrays | 42.336 | 4.113 | 0.748 | +| Arrow IPC | 44.005 | 3.446 | 0.448 | The backend paths and measured phase boundaries differ. This small smoke case validates parity and timing instrumentation, not general throughput. diff --git a/benchmark/fm-payloads/reports/omnistation-smoke.json b/benchmark/fm-payloads/reports/omnistation-smoke.json index 2d6f173f5..421198018 100644 --- a/benchmark/fm-payloads/reports/omnistation-smoke.json +++ b/benchmark/fm-payloads/reports/omnistation-smoke.json @@ -1,13 +1,14 @@ { "bundle_manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", - "workload_sha256": "df440df955a2edcae0f64acc5b5061272afd855006f944afbc0108a0b0eaf3c0", + "workload_sha256": "5c479c8c9438f297c23e5f8e0ce00ff95fc406f36bfe765c94723f75dbed0395", "wheel_sha256": { - "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "c55014ca6868058f9ddbd4efdcc57032eba24e9152dc445e6e4d95e56edc5818" + "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "f1146c4f079ca02a667da2a32fe6c62f91ea1f395cbf11fbd93f5ba1ad0a2701", + "diskgraph-0.2.0-cp311-abi3-linux_x86_64.whl": "ede0ca14dee5ccc75266afaf55c8ffd90f73a6e6f835448ddeff87655186522e" }, "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", - "engine_revision": "9637e9cc9e2da877959b3c6b9db88c0fc7e35f3f", + "engine_revision": "8d9e857f1bc020974787bd139f11444c481a88ba", "reference_revision": "b4516394e98929c5b4752745866d5e3138276255", - "checkout_dirty": true, + "checkout_dirty": false, "machine": { "platform": "Linux-7.0.0-1010-aws-x86_64-with-glibc2.39", "processor": "x86_64", @@ -32,29 +33,29 @@ "reference_body_bytes": 3374, "draws": [ { - "reference_full_ms": 41.190471, - "new_load_ms": 3.740637, - "new_encode_ms": 0.731258 + "reference_full_ms": 41.670437, + "new_load_ms": 4.9733, + "new_encode_ms": 1.11912 }, { - "reference_full_ms": 40.420899, - "new_load_ms": 3.376303, - "new_encode_ms": 0.704164 + "reference_full_ms": 41.872574, + "new_load_ms": 4.580137, + "new_encode_ms": 0.921378 }, { - "reference_full_ms": 40.677103, - "new_load_ms": 3.644739, - "new_encode_ms": 0.743447 + "reference_full_ms": 42.33557, + "new_load_ms": 4.032561, + "new_encode_ms": 0.747666 }, { - "reference_full_ms": 41.676955, - "new_load_ms": 3.449625, - "new_encode_ms": 0.73143 + "reference_full_ms": 42.599067, + "new_load_ms": 3.777073, + "new_encode_ms": 0.746791 }, { - "reference_full_ms": 40.691307, - "new_load_ms": 3.642878, - "new_encode_ms": 0.74569 + "reference_full_ms": 43.801003, + "new_load_ms": 4.112989, + "new_encode_ms": 0.699544 } ] }, @@ -67,29 +68,29 @@ "reference_body_bytes": 8859, "draws": [ { - "reference_full_ms": 41.208853, - "new_load_ms": 3.451474, - "new_encode_ms": 0.460802 + "reference_full_ms": 43.303153, + "new_load_ms": 3.446238, + "new_encode_ms": 0.464854 }, { - "reference_full_ms": 39.633872, - "new_load_ms": 3.457388, - "new_encode_ms": 0.443918 + "reference_full_ms": 44.005338, + "new_load_ms": 3.399719, + "new_encode_ms": 0.419488 }, { - "reference_full_ms": 40.712534, - "new_load_ms": 3.589053, - "new_encode_ms": 0.422048 + "reference_full_ms": 45.790362, + "new_load_ms": 4.089137, + "new_encode_ms": 0.448379 }, { - "reference_full_ms": 42.041882, - "new_load_ms": 3.405239, - "new_encode_ms": 0.445812 + "reference_full_ms": 46.826453, + "new_load_ms": 3.618977, + "new_encode_ms": 0.45852 }, { - "reference_full_ms": 40.765063, - "new_load_ms": 3.608553, - "new_encode_ms": 0.46231 + "reference_full_ms": 41.087149, + "new_load_ms": 3.432366, + "new_encode_ms": 0.428766 } ] } @@ -116,7 +117,7 @@ }, "eligibility": "supported", "exclusion_reasons": [], - "index_marker_sha256": "58ff9e961cf4bf9ed58598f4ae8ea483c3a0af5e0a819efdc5679b90eec236d5", + "index_marker_sha256": "0ada8dd747a380a67199e9d6138f02178a4f926c2a90ffebea9468fab2d4f087", "selected_rows": { "context": { "interactions": 2, From 5cc3efb54a86f7bf05a90d065dedfe3e8e5a1d35 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:11:28 +0000 Subject: [PATCH 07/12] Include FM manifest in public setup Docker context --- tests/public_setup/Dockerfile.dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/public_setup/Dockerfile.dockerignore b/tests/public_setup/Dockerfile.dockerignore index 598f31e29..67897beea 100644 --- a/tests/public_setup/Dockerfile.dockerignore +++ b/tests/public_setup/Dockerfile.dockerignore @@ -16,6 +16,7 @@ !bin/kumo-pred !packages/diskgraph/pyproject.toml !packages/experimental/kumo-pytorch/pyproject.toml +!packages/experimental/kumo-fm/pyproject.toml !tools/glove-6b-50d/pyproject.toml # Public fake accounts and policies for the local storage fixture. !docker/compose/config/minio-test/** From 51a6347bc27a3f80eb92d18d3bc37b41702a7a95 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:13:40 +0000 Subject: [PATCH 08/12] Apply repository license and workflow lint conventions --- .github/workflows/kumo-fm.yml | 6 +++++- benchmark/fm-payloads/make_fixture.py | 4 ++++ benchmark/fm-payloads/runner.py | 4 ++++ packages/experimental/kumo-fm/MANIFEST.in | 4 ++++ packages/experimental/kumo-fm/examples/local_sdm/adapter.py | 4 ++++ .../kumo-fm/examples/local_sdm/connector_to_prediction.py | 4 ++++ packages/experimental/kumo-fm/pyproject.toml | 4 ++++ packages/experimental/kumo-fm/setup.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/__init__.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_assembly.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_contract.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/__init__.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/config.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/generate.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/relbench.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/storage.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/tasks.py | 4 ++++ .../kumo-fm/src/kumo/fm/_dataset_support/validate.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_encoding.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_phase.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/_validation.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/batch.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/bundle.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/datasets.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/loader.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/native_json.py | 4 ++++ .../experimental/kumo-fm/src/kumo/fm/native_selection.py | 4 ++++ packages/experimental/kumo-fm/src/kumo/fm/workload.py | 4 ++++ packages/experimental/kumo-fm/tests/_fixture.py | 4 ++++ packages/experimental/kumo-fm/tests/test_batch.py | 4 ++++ .../experimental/kumo-fm/tests/test_connector_snapshot.py | 4 ++++ packages/experimental/kumo-fm/tests/test_contract.py | 4 ++++ packages/experimental/kumo-fm/tests/test_integration.py | 4 ++++ packages/experimental/kumo-fm/tests/test_sdm_adapter.py | 4 ++++ packages/experimental/kumo-fm/tests/test_snapshot_api.py | 4 ++++ .../experimental/kumo-fm/tests/test_snapshot_integration.py | 4 ++++ 38 files changed, 153 insertions(+), 1 deletion(-) diff --git a/.github/workflows/kumo-fm.yml b/.github/workflows/kumo-fm.yml index 42d8e33a4..c79ba863b 100644 --- a/.github/workflows/kumo-fm.yml +++ b/.github/workflows/kumo-fm.yml @@ -1,6 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + name: kumo-fm -on: +on: # yamllint disable-line rule:truthy pull_request: paths: - 'packages/experimental/kumo-fm/**' diff --git a/benchmark/fm-payloads/make_fixture.py b/benchmark/fm-payloads/make_fixture.py index 61cdd0b92..08671bc28 100644 --- a/benchmark/fm-payloads/make_fixture.py +++ b/benchmark/fm-payloads/make_fixture.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Create the frozen smoke case used by the benchmark and SDM notebook.""" import json diff --git a/benchmark/fm-payloads/runner.py b/benchmark/fm-payloads/runner.py index d1c8965e9..23ad3e469 100644 --- a/benchmark/fm-payloads/runner.py +++ b/benchmark/fm-payloads/runner.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Correctness-gated, interleaved FM payload timing against the DuckDB reference.""" import argparse import hashlib diff --git a/packages/experimental/kumo-fm/MANIFEST.in b/packages/experimental/kumo-fm/MANIFEST.in index f87b568a3..19ded0427 100644 --- a/packages/experimental/kumo-fm/MANIFEST.in +++ b/packages/experimental/kumo-fm/MANIFEST.in @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + recursive-include examples/local_sdm *.py *.ipynb *.md *.lock recursive-include examples/local_sdm/fixture *.json *.parquet recursive-include tests *.py diff --git a/packages/experimental/kumo-fm/examples/local_sdm/adapter.py b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py index 9e7fbdc8a..aee1efc6a 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/adapter.py +++ b/packages/experimental/kumo-fm/examples/local_sdm/adapter.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Example-local conversion from a trusted FM Arrow batch to public SDM inputs.""" import pyarrow as pa diff --git a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py index 2c07fdd1b..e17b9c076 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py +++ b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Connector → snapshot → retained FM Arrow batch → local SDM prediction. Run from this directory after installing the matching engine/native wheels and diff --git a/packages/experimental/kumo-fm/pyproject.toml b/packages/experimental/kumo-fm/pyproject.toml index b8be63e92..dc7d40562 100644 --- a/packages/experimental/kumo-fm/pyproject.toml +++ b/packages/experimental/kumo-fm/pyproject.toml @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + [build-system] requires = ["setuptools>=77", "pyarrow==24.0.0"] build-backend = "setuptools.build_meta" diff --git a/packages/experimental/kumo-fm/setup.py b/packages/experimental/kumo-fm/setup.py index dacd71c6f..ccc6cdbbe 100644 --- a/packages/experimental/kumo-fm/setup.py +++ b/packages/experimental/kumo-fm/setup.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import pyarrow from setuptools import Extension, setup diff --git a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py index 73c6ba1f1..0a1a60725 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/__init__.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/__init__.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """FM payload assembly through public Kumo PyTorch columnar batches.""" from .batch import PayloadBatch from .loader import FMLoader, PreparedRequest, prepare diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py index bacad08ba..cfca690b8 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Validate and assemble selected columnar graph occurrences into FM tables.""" from collections import defaultdict diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py index a557288da..5583f3444 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """FM schema and task metadata shared by direct and wire output.""" def schema_for(workload): diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py index 3c987464f..317e46a9a 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/__init__.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Portable, deterministic synthetic relational Parquet bundles.""" __version__ = "1.1.0" diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py index 6513f2015..0c17003d2 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/config.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Resolve small declarative presets; reject unsupported knobs before writing.""" import json diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py index de8c2c560..ad4dea4d8 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/generate.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Stateless vectorized generation and bounded Parquet writing.""" import hashlib diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py index 7578a2c26..6105ed76e 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Pinned, streaming adapters for the original RelBench F1 and H&M databases.""" import json diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py index ed2e07510..002b56bd8 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Batch-independent logical hashes and portable artifact inventories.""" import hashlib diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py index c3d0c3c02..319b1c40f 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/tasks.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Freeze cohorts, context, and future labels outside payload timing.""" from datetime import UTC, datetime diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py index e042be357..f9c9b974a 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/validate.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Streaming content checks plus spillable SQL integrity/label validation.""" import json diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py index b9dbcd298..068638049 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_encoding.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """HTTP encodings over retained Arrow tables.""" import base64 diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_phase.py b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py index b2e919ade..5239a18b8 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_phase.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_phase.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Small phase boundary for shared assembly code.""" from contextlib import nullcontext diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py index d29f7b34a..363aadefb 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_pytorch.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Public kumo.pytorch columnar adapter backed by DiskGraph.""" import hashlib import importlib.metadata diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py index ab0c8140f..eebb1ffa2 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_snapshot.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """FM orchestration over an already indexed public PyTorch engine.""" from __future__ import annotations diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_validation.py b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py index 59c3f86d7..95961b3d5 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_validation.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_validation.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Preparation-time native sampling compatibility checks.""" import duckdb diff --git a/packages/experimental/kumo-fm/src/kumo/fm/batch.py b/packages/experimental/kumo-fm/src/kumo/fm/batch.py index 27947918d..60e735a6e 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/batch.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/batch.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Retained FM Arrow tables and explicit HTTP encoding.""" from __future__ import annotations diff --git a/packages/experimental/kumo-fm/src/kumo/fm/bundle.py b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py index c5c8db8f9..270203af7 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/bundle.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/bundle.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Validated Parquet bundle metadata, shared by every backend. Full validation uses streaming file hashes and spillable DuckDB constraints; diff --git a/packages/experimental/kumo-fm/src/kumo/fm/datasets.py b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py index 60cc78ba0..e23e897fa 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/datasets.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/datasets.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Adapt fm_datasets' published bundles without rewriting their source artifacts.""" from copy import deepcopy diff --git a/packages/experimental/kumo-fm/src/kumo/fm/loader.py b/packages/experimental/kumo-fm/src/kumo/fm/loader.py index e8fc0d089..ac685b6ed 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/loader.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/loader.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Dataset, index, request, and batch lifecycle.""" from __future__ import annotations diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_json.py b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py index 7be2d2b53..1a314fbde 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/native_json.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_json.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Shared native arrays encoding; Python handles the small request envelope.""" from dataclasses import dataclass import json diff --git a/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py index 86ccdc17a..11960b5ee 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/native_selection.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Inclusive whole-second graph cutoff for native anchor tables.""" from datetime import timedelta diff --git a/packages/experimental/kumo-fm/src/kumo/fm/workload.py b/packages/experimental/kumo-fm/src/kumo/fm/workload.py index 2bbe7f4cc..5a196640c 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/workload.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/workload.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Frozen task instances and an explicit, bounded traversal policy.""" from __future__ import annotations diff --git a/packages/experimental/kumo-fm/tests/_fixture.py b/packages/experimental/kumo-fm/tests/_fixture.py index 52458eccd..014a16dde 100644 --- a/packages/experimental/kumo-fm/tests/_fixture.py +++ b/packages/experimental/kumo-fm/tests/_fixture.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Tiny whole-second contract bundles, not a bulk generator. Timestamp values have one-second granularity. Parquet and FM serialization diff --git a/packages/experimental/kumo-fm/tests/test_batch.py b/packages/experimental/kumo-fm/tests/test_batch.py index 2410d7ac5..359d1cacd 100644 --- a/packages/experimental/kumo-fm/tests/test_batch.py +++ b/packages/experimental/kumo-fm/tests/test_batch.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import base64 import json from datetime import datetime, timezone diff --git a/packages/experimental/kumo-fm/tests/test_connector_snapshot.py b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py index 6416d6279..4ac2eb89f 100644 --- a/packages/experimental/kumo-fm/tests/test_connector_snapshot.py +++ b/packages/experimental/kumo-fm/tests/test_connector_snapshot.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Installed-wheel connector → snapshot → FM, including exported reuse.""" import base64 import hashlib diff --git a/packages/experimental/kumo-fm/tests/test_contract.py b/packages/experimental/kumo-fm/tests/test_contract.py index f5091872b..3203de9a8 100644 --- a/packages/experimental/kumo-fm/tests/test_contract.py +++ b/packages/experimental/kumo-fm/tests/test_contract.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + from types import SimpleNamespace import pytest diff --git a/packages/experimental/kumo-fm/tests/test_integration.py b/packages/experimental/kumo-fm/tests/test_integration.py index bf323ad09..fec1e04aa 100644 --- a/packages/experimental/kumo-fm/tests/test_integration.py +++ b/packages/experimental/kumo-fm/tests/test_integration.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import json from copy import deepcopy from datetime import timedelta diff --git a/packages/experimental/kumo-fm/tests/test_sdm_adapter.py b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py index 563ff39c3..3584762ad 100644 --- a/packages/experimental/kumo-fm/tests/test_sdm_adapter.py +++ b/packages/experimental/kumo-fm/tests/test_sdm_adapter.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + import sys from pathlib import Path from types import SimpleNamespace diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_api.py b/packages/experimental/kumo-fm/tests/test_snapshot_api.py index c272f2411..479967ba8 100644 --- a/packages/experimental/kumo-fm/tests/test_snapshot_api.py +++ b/packages/experimental/kumo-fm/tests/test_snapshot_api.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Snapshot input validation before any native graph is sampled.""" from datetime import datetime, timezone from types import SimpleNamespace diff --git a/packages/experimental/kumo-fm/tests/test_snapshot_integration.py b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py index c79294c2d..b5bc02466 100644 --- a/packages/experimental/kumo-fm/tests/test_snapshot_integration.py +++ b/packages/experimental/kumo-fm/tests/test_snapshot_integration.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + """Integrated multi-layer FM sampling over a reusable native index.""" from datetime import datetime, timezone From 843b3377e6c1fbf831bc56eede2f79891155a791 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:14:25 +0000 Subject: [PATCH 09/12] Record smoke comparison with final licensed wheel --- benchmark/fm-payloads/README.md | 4 +- .../reports/omnistation-smoke.json | 64 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md index 61f1bc7ec..7c78bcade 100644 --- a/benchmark/fm-payloads/README.md +++ b/benchmark/fm-payloads/README.md @@ -37,8 +37,8 @@ counts. Median phase times in milliseconds: | Encoding | DuckDB reference full payload | DiskGraph load and assembly | Encoding retained batch | | --- | ---: | ---: | ---: | -| Arrays | 42.336 | 4.113 | 0.748 | -| Arrow IPC | 44.005 | 3.446 | 0.448 | +| Arrays | 41.228 | 3.541 | 0.720 | +| Arrow IPC | 40.517 | 3.549 | 0.440 | The backend paths and measured phase boundaries differ. This small smoke case validates parity and timing instrumentation, not general throughput. diff --git a/benchmark/fm-payloads/reports/omnistation-smoke.json b/benchmark/fm-payloads/reports/omnistation-smoke.json index 421198018..2dbdcd308 100644 --- a/benchmark/fm-payloads/reports/omnistation-smoke.json +++ b/benchmark/fm-payloads/reports/omnistation-smoke.json @@ -2,11 +2,11 @@ "bundle_manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", "workload_sha256": "5c479c8c9438f297c23e5f8e0ce00ff95fc406f36bfe765c94723f75dbed0395", "wheel_sha256": { - "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "f1146c4f079ca02a667da2a32fe6c62f91ea1f395cbf11fbd93f5ba1ad0a2701", + "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "71ccf8a984f5383738d41bfcf0cbd00613afd6059142fbbc4ab83c26df35750f", "diskgraph-0.2.0-cp311-abi3-linux_x86_64.whl": "ede0ca14dee5ccc75266afaf55c8ffd90f73a6e6f835448ddeff87655186522e" }, "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", - "engine_revision": "8d9e857f1bc020974787bd139f11444c481a88ba", + "engine_revision": "51a6347bc27a3f80eb92d18d3bc37b41702a7a95", "reference_revision": "b4516394e98929c5b4752745866d5e3138276255", "checkout_dirty": false, "machine": { @@ -33,29 +33,29 @@ "reference_body_bytes": 3374, "draws": [ { - "reference_full_ms": 41.670437, - "new_load_ms": 4.9733, - "new_encode_ms": 1.11912 + "reference_full_ms": 40.174507, + "new_load_ms": 3.496997, + "new_encode_ms": 0.718417 }, { - "reference_full_ms": 41.872574, - "new_load_ms": 4.580137, - "new_encode_ms": 0.921378 + "reference_full_ms": 40.663971, + "new_load_ms": 3.771849, + "new_encode_ms": 0.737373 }, { - "reference_full_ms": 42.33557, - "new_load_ms": 4.032561, - "new_encode_ms": 0.747666 + "reference_full_ms": 41.900469, + "new_load_ms": 3.465119, + "new_encode_ms": 0.745808 }, { - "reference_full_ms": 42.599067, - "new_load_ms": 3.777073, - "new_encode_ms": 0.746791 + "reference_full_ms": 41.228331, + "new_load_ms": 3.540971, + "new_encode_ms": 0.720256 }, { - "reference_full_ms": 43.801003, - "new_load_ms": 4.112989, - "new_encode_ms": 0.699544 + "reference_full_ms": 41.340062, + "new_load_ms": 3.618949, + "new_encode_ms": 0.71508 } ] }, @@ -68,29 +68,29 @@ "reference_body_bytes": 8859, "draws": [ { - "reference_full_ms": 43.303153, - "new_load_ms": 3.446238, - "new_encode_ms": 0.464854 + "reference_full_ms": 39.87662, + "new_load_ms": 3.790806, + "new_encode_ms": 0.439622 }, { - "reference_full_ms": 44.005338, - "new_load_ms": 3.399719, - "new_encode_ms": 0.419488 + "reference_full_ms": 39.513152, + "new_load_ms": 3.548801, + "new_encode_ms": 0.447506 }, { - "reference_full_ms": 45.790362, - "new_load_ms": 4.089137, - "new_encode_ms": 0.448379 + "reference_full_ms": 40.953171, + "new_load_ms": 3.881504, + "new_encode_ms": 0.428711 }, { - "reference_full_ms": 46.826453, - "new_load_ms": 3.618977, - "new_encode_ms": 0.45852 + "reference_full_ms": 40.516572, + "new_load_ms": 3.458438, + "new_encode_ms": 0.424711 }, { - "reference_full_ms": 41.087149, - "new_load_ms": 3.432366, - "new_encode_ms": 0.428766 + "reference_full_ms": 40.77798, + "new_load_ms": 3.446517, + "new_encode_ms": 0.476014 } ] } From 539e9a4f1cb7ae7cfa254ea2eac90629a07a032b Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:19:11 +0000 Subject: [PATCH 10/12] Align FM sources and licensing inventory with repository checks --- benchmark/fm-payloads/runner.py | 4 +- .../local_sdm/connector_to_prediction.py | 11 +++-- .../kumo-fm/src/kumo/fm/_assembly.py | 2 +- .../kumo-fm/src/kumo/fm/_contract.py | 2 +- .../src/kumo/fm/_dataset_support/relbench.py | 1 + .../src/kumo/fm/_dataset_support/storage.py | 4 +- .../kumo-fm/src/kumo/fm/_encoding.py | 4 +- .../kumo-fm/src/kumo/fm/_pytorch.py | 19 ++++----- .../kumo-fm/src/kumo/fm/_snapshot.py | 25 ++++++------ .../kumo-fm/src/kumo/fm/_validation.py | 3 +- .../experimental/kumo-fm/src/kumo/fm/batch.py | 4 +- .../kumo-fm/src/kumo/fm/loader.py | 4 +- .../kumo-fm/src/kumo/fm/native_json.py | 4 +- .../kumo-fm/src/kumo/fm/workload.py | 4 +- .../experimental/kumo-fm/tests/_fixture.py | 12 +++--- .../experimental/kumo-fm/tests/test_batch.py | 2 +- .../kumo-fm/tests/test_connector_snapshot.py | 29 ++++++++------ .../kumo-fm/tests/test_integration.py | 3 +- .../kumo-fm/tests/test_sdm_adapter.py | 3 +- .../kumo-fm/tests/test_snapshot_api.py | 40 +++++++++++-------- .../tests/test_snapshot_integration.py | 26 ++++++------ scripts/check_python_license_metadata.py | 2 + scripts/gen_third_party.py | 1 + test/scripts/test_python_license_metadata.py | 2 +- 24 files changed, 110 insertions(+), 101 deletions(-) diff --git a/benchmark/fm-payloads/runner.py b/benchmark/fm-payloads/runner.py index 23ad3e469..c150c9d35 100644 --- a/benchmark/fm-payloads/runner.py +++ b/benchmark/fm-payloads/runner.py @@ -11,13 +11,13 @@ import random import subprocess import time -from urllib.parse import urlparse from pathlib import Path +from urllib.parse import urlparse from kumo import fm +from kumo.fm._pytorch import compatibility_issues from kumo.fm.bundle import Bundle from kumo.fm.workload import Workload -from kumo.fm._pytorch import compatibility_issues def revision(path): diff --git a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py index e17b9c076..242c9213c 100644 --- a/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py +++ b/packages/experimental/kumo-fm/examples/local_sdm/connector_to_prediction.py @@ -10,16 +10,15 @@ from datetime import datetime, timezone from pathlib import Path -import kumo.engine as kumo -from kumo.engine import datasets -from kumo.pytorch import api -from kumoapi.typing import Stype import torch +from adapter import arrow_to_sdm +from kumoapi.typing import Stype from sdm.models import KumoRelational +import kumo.engine as kumo from kumo import fm -from adapter import arrow_to_sdm - +from kumo.engine import datasets +from kumo.pytorch import api root = Path("scratch/connector-to-prediction").resolve() root.mkdir(parents=True, exist_ok=True) diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py index cfca690b8..14e680031 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_assembly.py @@ -9,8 +9,8 @@ import pyarrow as pa import pyarrow.compute as pc -from .bundle import TYPES from ._phase import phase +from .bundle import TYPES def empty_features(columns): diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py index 5583f3444..ba0273a8a 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_contract.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_contract.py @@ -4,6 +4,7 @@ """FM schema and task metadata shared by direct and wire output.""" + def schema_for(workload): bundle = workload.bundle id_spec = {"dtype": "int64", "stype": "ID", "nullable": False} @@ -42,7 +43,6 @@ def schema_for(workload): return {"instance_table": instance, "related_tables": related, "relationships": relationships} - def metadata_for(workload): target = dict(workload.target) if workload.kind != "regression": diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py index 6105ed76e..3fb76bc42 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/relbench.py @@ -134,6 +134,7 @@ def import_relbench(root, cache, dataset, config=None): cache.mkdir(parents=True, exist_ok=True) source_budget = Budget(cache, cfg) source_budget.check() + def get(relative): return fetch(cache, relative, source_budget) source_manifest = get(f"{dataset}/manifest.yaml") diff --git a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py index 002b56bd8..d20d5afc8 100644 --- a/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py +++ b/packages/experimental/kumo-fm/src/kumo/fm/_dataset_support/storage.py @@ -59,11 +59,11 @@ def update(self, batch): if pa.types.is_string(field.type): filled = pc.fill_null(array, "") offsets = np.frombuffer(filled.buffers()[1], dtype=" None: - assert len(check_python_license_metadata.DISTRIBUTIONS) == 15 + assert len(check_python_license_metadata.DISTRIBUTIONS) == 16 assert check_python_license_metadata.validate_repository( REPOSITORY_ROOT) == [] From 6d8be74aed7a3c472fb4185462aef4ab667bad8e Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:19:58 +0000 Subject: [PATCH 11/12] Refresh frozen comparison after lint cleanup --- benchmark/fm-payloads/README.md | 4 +- .../reports/omnistation-smoke.json | 64 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/benchmark/fm-payloads/README.md b/benchmark/fm-payloads/README.md index 7c78bcade..7f2b42d3b 100644 --- a/benchmark/fm-payloads/README.md +++ b/benchmark/fm-payloads/README.md @@ -37,8 +37,8 @@ counts. Median phase times in milliseconds: | Encoding | DuckDB reference full payload | DiskGraph load and assembly | Encoding retained batch | | --- | ---: | ---: | ---: | -| Arrays | 41.228 | 3.541 | 0.720 | -| Arrow IPC | 40.517 | 3.549 | 0.440 | +| Arrays | 40.200 | 3.453 | 0.710 | +| Arrow IPC | 40.141 | 3.477 | 0.424 | The backend paths and measured phase boundaries differ. This small smoke case validates parity and timing instrumentation, not general throughput. diff --git a/benchmark/fm-payloads/reports/omnistation-smoke.json b/benchmark/fm-payloads/reports/omnistation-smoke.json index 2dbdcd308..db8081a16 100644 --- a/benchmark/fm-payloads/reports/omnistation-smoke.json +++ b/benchmark/fm-payloads/reports/omnistation-smoke.json @@ -2,11 +2,11 @@ "bundle_manifest_sha256": "756537788732b93e1c94f24c386cf253d867adc5220e4689d636c1627a88467b", "workload_sha256": "5c479c8c9438f297c23e5f8e0ce00ff95fc406f36bfe765c94723f75dbed0395", "wheel_sha256": { - "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "71ccf8a984f5383738d41bfcf0cbd00613afd6059142fbbc4ab83c26df35750f", + "kumo_fm-0.1.0a1-cp311-cp311-linux_x86_64.whl": "92cd45fb8691db74c86ab53f382a2880231008f2db02ba1de4ae77a259479eb6", "diskgraph-0.2.0-cp311-abi3-linux_x86_64.whl": "ede0ca14dee5ccc75266afaf55c8ffd90f73a6e6f835448ddeff87655186522e" }, "snapshot_id": "7a69e955e84d54d6b3b2711be57f9bf1569ada00abbae1a658fb31b0cfdaa76e", - "engine_revision": "51a6347bc27a3f80eb92d18d3bc37b41702a7a95", + "engine_revision": "539e9a4f1cb7ae7cfa254ea2eac90629a07a032b", "reference_revision": "b4516394e98929c5b4752745866d5e3138276255", "checkout_dirty": false, "machine": { @@ -33,29 +33,29 @@ "reference_body_bytes": 3374, "draws": [ { - "reference_full_ms": 40.174507, - "new_load_ms": 3.496997, - "new_encode_ms": 0.718417 + "reference_full_ms": 41.671234, + "new_load_ms": 3.360692, + "new_encode_ms": 0.694896 }, { - "reference_full_ms": 40.663971, - "new_load_ms": 3.771849, - "new_encode_ms": 0.737373 + "reference_full_ms": 39.565126, + "new_load_ms": 3.484977, + "new_encode_ms": 0.713286 }, { - "reference_full_ms": 41.900469, - "new_load_ms": 3.465119, - "new_encode_ms": 0.745808 + "reference_full_ms": 40.508425, + "new_load_ms": 3.447152, + "new_encode_ms": 0.710074 }, { - "reference_full_ms": 41.228331, - "new_load_ms": 3.540971, - "new_encode_ms": 0.720256 + "reference_full_ms": 40.103816, + "new_load_ms": 3.452723, + "new_encode_ms": 0.979746 }, { - "reference_full_ms": 41.340062, - "new_load_ms": 3.618949, - "new_encode_ms": 0.71508 + "reference_full_ms": 40.199899, + "new_load_ms": 3.533769, + "new_encode_ms": 0.698663 } ] }, @@ -68,29 +68,29 @@ "reference_body_bytes": 8859, "draws": [ { - "reference_full_ms": 39.87662, - "new_load_ms": 3.790806, - "new_encode_ms": 0.439622 + "reference_full_ms": 40.14074, + "new_load_ms": 3.622619, + "new_encode_ms": 0.465381 }, { - "reference_full_ms": 39.513152, - "new_load_ms": 3.548801, - "new_encode_ms": 0.447506 + "reference_full_ms": 41.85428, + "new_load_ms": 3.477252, + "new_encode_ms": 0.424267 }, { - "reference_full_ms": 40.953171, - "new_load_ms": 3.881504, - "new_encode_ms": 0.428711 + "reference_full_ms": 40.112547, + "new_load_ms": 3.372056, + "new_encode_ms": 0.444124 }, { - "reference_full_ms": 40.516572, - "new_load_ms": 3.458438, - "new_encode_ms": 0.424711 + "reference_full_ms": 41.194207, + "new_load_ms": 3.428842, + "new_encode_ms": 0.422983 }, { - "reference_full_ms": 40.77798, - "new_load_ms": 3.446517, - "new_encode_ms": 0.476014 + "reference_full_ms": 40.038979, + "new_load_ms": 3.915465, + "new_encode_ms": 0.420586 } ] } From e998187d4fdee4289a814d97b947e74effe3fd47 Mon Sep 17 00:00:00 2001 From: Alex Porter Date: Thu, 19 Nov 2026 01:20:20 +0000 Subject: [PATCH 12/12] Match YAML comment spacing in FM workflow --- .github/workflows/kumo-fm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/kumo-fm.yml b/.github/workflows/kumo-fm.yml index c79ba863b..28a9e6a21 100644 --- a/.github/workflows/kumo-fm.yml +++ b/.github/workflows/kumo-fm.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 45 steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: persist-credentials: false - uses: ./.github/actions/kumo-pytorch-setup