From a8ee9f223467856795a8288e24eac502f8d7c39b Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 11:44:53 -0700 Subject: [PATCH 01/39] Support macOS engine wheels and Metal retrieval --- docs/development/releasing-kumo-engine.md | 11 ++ packages/kumo-core/README.md | 14 ++ packages/kumo-core/pyproject.toml | 2 +- .../src/kumo_core/evaluation/suite.py | 12 ++ .../src/kumo_core/modeling/retrieval/mips.py | 50 ++++- .../src/kumo_core/train/base_module.py | 8 +- .../kumo-core/tests/evaluation/test_suite.py | 90 +++++++++ .../tests/modeling/retrieval/test_mips.py | 187 +++++++++++++++++- packages/kumo-engine/pyproject.toml | 4 +- packages/kumo-engine/scripts/build_wheel.sh | 55 ++++-- packages/kumo-engine/scripts/merge_wheels.py | 11 ++ packages/kumo-engine/setup.py | 15 +- test/scripts/test_engine_wheel.py | 84 +++++++- uv.lock | 30 +-- 14 files changed, 519 insertions(+), 54 deletions(-) diff --git a/docs/development/releasing-kumo-engine.md b/docs/development/releasing-kumo-engine.md index c0f710305..20487de0f 100644 --- a/docs/development/releasing-kumo-engine.md +++ b/docs/development/releasing-kumo-engine.md @@ -29,6 +29,11 @@ uses by default to encode text columns; without it, a text column cannot be encoded and the error asks for `pip install "kumo-engine[text]"`. `torch-geometric>=2.8.1,<3` is also resolved separately at install time. +Faiss is selected at install time: `faiss-gpu==1.14.3` on Linux and +`faiss-cpu==1.14.3` on macOS. These markers live on the engine's direct +dependencies so merging bundled packages preserves them. This dependency +selection does not establish macOS wheel or runtime support. + Releases are cut from `NVIDIA/kumo-engine`: its GitHub releases trigger the workflows below, which attach the wheels to the release. Their `kitmaker` job then has Kitmaker, NVIDIA's release service, fetch the attached wheel and @@ -153,6 +158,12 @@ access for component build dependencies. `--verify-image IMAGE` installs the wheel into a Docker image and checks that the closure imports and that provider dependencies are absent from a bare install. +On Apple Silicon, the same script builds a native macOS wheel with +`MACOSX_DEPLOYMENT_TARGET=14.0` by default. It disables Zig and glibc checks +and takes the final platform tag from the built DiskGraph wheel. Rust, Xcode +Command Line Tools, and CMake are required. This local build path does not +add macOS publication to the release workflow. + ## PyPI projects `kumo-engine` and `kumo-glove-6b-50d` belong to NVIDIA's PyPI organization diff --git a/packages/kumo-core/README.md b/packages/kumo-core/README.md index fe5494005..33d4e62db 100644 --- a/packages/kumo-core/README.md +++ b/packages/kumo-core/README.md @@ -58,6 +58,20 @@ Materialization accepts the selected graph in `TrainingWorkflowRequest`, a `Task Use `kumo_core.types` for value types, `kumo_core.training` for workflow contracts, and `kumo_core.train` for the numerical stack. These initializers expose deliberate lazy APIs without loading PyTorch during package discovery. +`MIPSFaiss` uses PyTorch's `MIPSBruteForce` for CPU tensors on macOS, avoiding +Faiss CPU search and its OpenMP runtime conflict with PyTorch. Faiss is loaded +only when a Faiss backend is needed. Brute-force search materializes the full +query-by-key score matrix, so large CPU workloads need additional memory. +Linux CPU and CUDA tensors retain their Faiss search paths. + +For MPS embeddings on Apple Silicon, `MIPSFaiss` uses Faiss's Metal backend. +It requires a Metal-enabled Faiss build, provided by `faiss-cpu==1.14.3` on +macOS ARM64. Keys are copied into the Metal index once; queries pass through +CPU float32 arrays, and scores and indices return to the original MPS device. +Scores retain the input dtype, exclusions are applied on MPS, and query +gradients are computed with PyTorch. This retrieval path does not establish +support for the full training pipeline on MPS. + ## Development and validation Run commands from the repository worktree after installing the normal Kumo development environment with `make install-venv`. `make test-kumo-core` runs the complete package-owned CPU suite, and `make check-kumo-core` runs Flake8's Pyflakes rules across the full package, ty across the migrated capability tree, and package metadata validation. The package remains temporarily excluded from the root YAPF/isort/full-Flake8 formatting policy pending the planned repository-wide Ruff migration. diff --git a/packages/kumo-core/pyproject.toml b/packages/kumo-core/pyproject.toml index 66165aab4..784d2d239 100644 --- a/packages/kumo-core/pyproject.toml +++ b/packages/kumo-core/pyproject.toml @@ -68,7 +68,7 @@ documentation = "https://kumo.ai/docs" # `uv sync --all-packages --extra cpu` applies a same-named extra to every # workspace member that defines one -- a bare "cpu" extra here would pull # in faiss-cpu alongside whatever faiss build root's own install chose. -faiss-cpu = ["faiss-cpu==1.14.2"] +faiss-cpu = ["faiss-cpu==1.14.3"] faiss-gpu = ["faiss-gpu==1.14.3"] text = ["kumo-glove-6b-50d>=1.0.0,<2"] test = [ diff --git a/packages/kumo-core/src/kumo_core/evaluation/suite.py b/packages/kumo-core/src/kumo_core/evaluation/suite.py index 96b083c8b..dd7fd9dc2 100644 --- a/packages/kumo-core/src/kumo_core/evaluation/suite.py +++ b/packages/kumo-core/src/kumo_core/evaluation/suite.py @@ -609,6 +609,18 @@ def get_metric_collection( stage: TrainingStage, ) -> torchmetrics.MetricCollection | LinkPredMetricCollection: metric_collection = self._metric_collections[f'{stage}_metrics'] + if isinstance(metric_collection, torchmetrics.MetricCollection): + for metric in metric_collection.values(copy_state=False): + if metric.device.type == 'mps': + # On macOS 26.7.1 / PyTorch 2.12.1, MPSGraph constant + # padding corrupts 1D tensors when the padded length + # reaches 65,536. TorchMetrics uses F.pad to append a + # threshold index for AP/AUPRC/AUROC, causing invalid + # metrics or out-of-bounds errors on large epochs. + # Keep accumulated prediction/target lists and their + # computation on CPU; model training stays on MPS. + # Check the actual metric device after model placement. + metric.compute_on_cpu = True if self.task_type == TaskType.MULTILABEL_CLASSIFICATION: metric_collection.cpu() # Use CPU for multilabel classification. return metric_collection diff --git a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py index fbda209f1..e16074dad 100644 --- a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py +++ b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py @@ -1,15 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import sys import warnings from abc import ABC, abstractmethod +from typing import TYPE_CHECKING -import faiss -import faiss.contrib.torch_utils import torch from torch import Tensor from torch_geometric.utils import scatter +if TYPE_CHECKING: + import faiss + import logging logger = logging.getLogger(__name__) @@ -127,15 +130,41 @@ def _search( class MIPSFaiss(MIPSStructure): - r"""MIPS based on ``faiss``.""" + r"""MIPS using PyTorch on macOS CPU and Faiss for other backends. + + Metal's Python API accepts host arrays, so inputs and search results + pass through CPU memory. Keys are copied into the Metal index once. + """ def __init__(self, key: Tensor) -> None: super().__init__(key=key) + self._cpu_index: MIPSBruteForce | None = None self._res: faiss.StandardGpuResources | None = None + self._metal_index: faiss.Index | None = None + if sys.platform == 'darwin' and self.device.type == 'cpu': + self._cpu_index = MIPSBruteForce(self._key) + return + + # CPU-only macOS processes must not load Faiss's second OpenMP runtime. + import faiss + import faiss.contrib.torch_utils + if self._key.is_cuda: self._res = faiss.StandardGpuResources() self._res.setDefaultNullStreamAllDevices() self._res.setTempMemory(0) + elif self.device.type == 'mps' and self.num_keys > 0: + if ('MAC_METAL' not in faiss.get_compile_options() + or faiss.get_num_gpus() == 0): + raise RuntimeError( + 'MPS retrieval requires a Metal-enabled Faiss build and ' + 'an available Metal device. Install faiss-cpu==1.14.3 ' + 'on Apple Silicon or use CPU tensors.') + self._res = faiss.StandardGpuResources() + self._metal_index = faiss.index_cpu_to_gpu( + self._res, 0, faiss.IndexFlatIP(self.embedding_dim)) + self._metal_index.add( + self._key.to(device='cpu', dtype=torch.float32).numpy()) def _search( self, @@ -144,6 +173,11 @@ def _search( exclude: tuple[Tensor, Tensor] | None, ) -> tuple[Tensor, Tensor]: + if self._cpu_index is not None: + return self._cpu_index.search(query, k, exclude) + + import faiss + query_k = k if exclude is not None: # Over-retrieve `k` to compensate for exclusions: @@ -162,10 +196,16 @@ def _search( f"retrieved.") query_k = MAX_K - # Zero-copy MIPS over PyTorch tensors via `faiss`: with warnings.catch_warnings(): warnings.filterwarnings('ignore', message='TypedStorage') - if self._res is not None: + if self._metal_index is not None: + score, index = self._metal_index.search( + query.to(device='cpu', dtype=torch.float32).numpy(), + query_k) + score = torch.from_numpy(score).to( + device=self.device, dtype=self.dtype) + index = torch.from_numpy(index).to(self.device) + elif self._res is not None: score, index = faiss.knn_gpu(self._res, query, self._key, query_k, metric=faiss.METRIC_INNER_PRODUCT) diff --git a/packages/kumo-core/src/kumo_core/train/base_module.py b/packages/kumo-core/src/kumo_core/train/base_module.py index 8ac41c615..568460fda 100644 --- a/packages/kumo-core/src/kumo_core/train/base_module.py +++ b/packages/kumo-core/src/kumo_core/train/base_module.py @@ -670,8 +670,12 @@ def compute_metrics(self, stage: TrainingStage) -> dict[str, Tensor]: metric.__getitem__(key, False).target = [target_cat[sampled_idx]] - # Use GPU for fast computation - metric.to(self.device) + # Multilabel predictions/targets are accumulated on CPU above. + # Moving them back to MPS here would expose large curves to the + # MPSGraph padding bug again: compute_on_cpu moves list states + # after update(), not after an explicit to(device) before compute(). + if self.device.type != 'mps': + metric.to(self.device) out_dict = metric.compute() metric.reset() diff --git a/packages/kumo-core/tests/evaluation/test_suite.py b/packages/kumo-core/tests/evaluation/test_suite.py index 28e827f46..a9ef5ca4d 100644 --- a/packages/kumo-core/tests/evaluation/test_suite.py +++ b/packages/kumo-core/tests/evaluation/test_suite.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from types import SimpleNamespace + import pytest import torch import torchmetrics @@ -115,6 +117,94 @@ def test_metric_store() -> None: assert set(m.keys()) == {f'{stage}_{metric}' for metric in metrics} +@pytest.mark.parametrize('device', ['cpu', 'cuda', 'mps']) +@pytest.mark.parametrize('stage', TrainingStage.list_stages()) +def test_large_classification_metrics( + device: str, + stage: TrainingStage, +) -> None: + if device == 'cuda' and not torch.cuda.is_available(): + pytest.skip('CUDA unavailable') + if device == 'mps' and not torch.backends.mps.is_available(): + pytest.skip('MPS unavailable') + + generator = torch.Generator().manual_seed(17) + # More than 2**16 distinct thresholds triggers the MPS padding bug. + pred = torch.rand(70_257, generator=generator) + target = torch.randint(2, (len(pred), ), generator=generator) + encoder = Index() + encoder._stats = {ColStatType.CATEGORY_COUNTS: [['A', 1], ['B', 1]]} + target_encoder = TargetEncoder.from_encoder( + TaskType.BINARY_CLASSIFICATION, encoder) + + def make_store(device: str) -> MetricStore: + return MetricStore( + TaskType.BINARY_CLASSIFICATION, target_encoder).init_metrics( + ['ap', 'auroc', 'auprc'], torch.nn.Module()).to(device) + + reference = make_store('cpu').get_metric_collection(stage) + reference.update(pred, target) + expected = reference.compute() + store = make_store(device) + for _ in range(2): + for batch_pred, batch_target in zip(pred.split(512), + target.split(512)): + metric = store.get_metric_collection(stage) + metric.update(batch_pred.to(device), batch_target.to(device)) + + metric = store.get_metric_collection(stage) + for member in metric.values(copy_state=False): + assert member.compute_on_cpu == (device == 'mps') + assert member.preds[0].device.type == ( + 'cpu' if device == 'mps' else device) + for name, value in metric.compute().items(): + torch.testing.assert_close(value.cpu(), expected[name]) + metric.reset() + + +@pytest.mark.parametrize('device', ['cpu', 'cuda', 'mps']) +def test_large_multilabel_metrics(device: str) -> None: + from kumo_core.train.base_module import BaseLightningModule + + if device == 'cuda' and not torch.cuda.is_available(): + pytest.skip('CUDA unavailable') + if device == 'mps' and not torch.backends.mps.is_available(): + pytest.skip('MPS unavailable') + + encoder = Index() + encoder._stats = {ColStatType.CATEGORY_COUNTS: [['A', 1], ['B', 1]]} + target_encoder = TargetEncoder.from_encoder( + TaskType.MULTILABEL_CLASSIFICATION, encoder) + generator = torch.Generator().manual_seed(17) + pred = torch.randn(70_257, 2, generator=generator) + target = torch.randint(2, pred.shape, generator=generator) + expected = None + for metric_device in ['cpu', device]: + store = MetricStore( + TaskType.MULTILABEL_CLASSIFICATION, target_encoder).init_metrics( + ['ap_macro', 'auroc_macro', 'auprc_macro'], + torch.nn.Module()).to(metric_device) + for batch_pred, batch_target in zip(pred.split(512), + target.split(512)): + store.get_metric_collection(TrainingStage.TRAIN).update( + batch_pred.to(metric_device), batch_target.to(metric_device)) + module = SimpleNamespace( + metric_store=store, + task_type=TaskType.MULTILABEL_CLASSIFICATION, + max_multilabel_samples=100_000_000, + device=torch.device(metric_device), + is_fitting=True, + log_epoch=lambda *args: None, + ) + actual = BaseLightningModule.compute_metrics(module, + TrainingStage.TRAIN) + if expected is None: + expected = actual + else: + for name, value in actual.items(): + torch.testing.assert_close(value.cpu(), expected[name]) + + def test_validate_diversity() -> None: stype_dict = { 'users': {}, diff --git a/packages/kumo-core/tests/modeling/retrieval/test_mips.py b/packages/kumo-core/tests/modeling/retrieval/test_mips.py index 8f251a499..b7dc2241e 100644 --- a/packages/kumo-core/tests/modeling/retrieval/test_mips.py +++ b/packages/kumo-core/tests/modeling/retrieval/test_mips.py @@ -1,15 +1,29 @@ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import os +import subprocess +import sys +from types import SimpleNamespace + import pytest import torch -from torch_geometric.testing import noMac, withCUDA -from kumo_core.modeling.retrieval.mips import MIPSBruteForce, MIPSFaiss +from kumo_core.modeling.retrieval.mips import MAX_K, MIPSBruteForce, MIPSFaiss + + +@pytest.fixture(params=[ + 'cpu', + pytest.param('cuda', marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason='CUDA unavailable')), + pytest.param('mps', marks=pytest.mark.skipif( + not torch.backends.mps.is_available(), reason='MPS unavailable')), +]) +def device(request: pytest.FixtureRequest) -> torch.device: + return torch.device(request.param if request.param == 'cpu' + else f'{request.param}:0') -@noMac -@withCUDA @pytest.mark.parametrize('dtype', [torch.float32]) @pytest.mark.parametrize('num_lhs', [10, 20]) @pytest.mark.parametrize('num_rhs', [100, 1_000, 10_000]) @@ -59,12 +73,13 @@ def test_mips_faiss( mat[exclude[0], exclude[1]] = -float('inf') expected = mat.argsort(dim=-1, descending=True)[:, :k] - # Only require non-inf prediction to match: + # Compare scores so equivalent near-tied neighbors need not share IDs. notinf = ~torch.isinf(score) - assert torch.allclose(index[notinf], expected[notinf]) + torch.testing.assert_close(mat.gather(1, index)[notinf], + mat.gather(1, expected)[notinf], + rtol=1e-5, atol=1e-5) -@withCUDA @pytest.mark.parametrize('dtype', [torch.float32]) @pytest.mark.parametrize('num_lhs', [10, 20]) @pytest.mark.parametrize('num_rhs', [100, 1_000, 10_000]) @@ -119,8 +134,6 @@ def test_mips_brute_force( assert torch.allclose(index[notinf], expected[notinf]) -@noMac -@withCUDA @pytest.mark.parametrize('cls', [MIPSBruteForce, MIPSFaiss]) def test_empty( cls: type[MIPSBruteForce | MIPSFaiss], @@ -137,3 +150,159 @@ def test_empty( mips_structure = cls(rhs) assert mips_structure.search(lhs, k=2)[0].size() == (0, 2) assert mips_structure.search(lhs, k=2)[1].size() == (0, 2) + + +@pytest.mark.skipif(not torch.backends.mps.is_available(), + reason='MPS unavailable') +@pytest.mark.parametrize('dtype', [torch.float32, torch.float16, + torch.bfloat16]) +@pytest.mark.parametrize('with_exclude', [False, True]) +def test_mips_metal_scores_and_gradients( + dtype: torch.dtype, + with_exclude: bool, +) -> None: + # Transposing twice with a copy gives non-contiguous input tensors. + rhs = torch.tensor([ + [1, 0, 0, 0], [0, 2, 0, 0], [0, 0, 3, 0], [0, 0, 0, 4], + [-1, -1, -1, -1], + ], device='mps', dtype=dtype).t().contiguous().t() + lhs = torch.tensor([[1, 2, 3, 4], [7, 5, 2, 1]], + device='mps', dtype=dtype).t().contiguous().t() + lhs.requires_grad_() + expected_lhs = lhs.detach().clone().requires_grad_() + exclude = None + if with_exclude: + exclude = (torch.tensor([0, 1], device='mps'), + torch.tensor([3, 1], device='mps')) + mips = MIPSFaiss(rhs) + assert mips._metal_index is not None + # Reuse the index across searches, including inference and autograd. + inference_score, inference_index = mips.search(lhs.detach(), 2, exclude) + score, index = mips.search(lhs, 2, exclude) + expected_score, expected_index = MIPSBruteForce(rhs).search( + expected_lhs, 2, exclude) + assert score.device == index.device == lhs.device + assert score.dtype == dtype + assert index.dtype == torch.long + torch.testing.assert_close(score, expected_score) + torch.testing.assert_close(index, expected_index) + torch.testing.assert_close(inference_score, score.detach()) + torch.testing.assert_close(inference_index, index) + score.sum().backward() + expected_score.sum().backward() + torch.testing.assert_close(lhs.grad, expected_lhs.grad) + + +@pytest.mark.skipif(not torch.backends.mps.is_available(), + reason='MPS unavailable') +@pytest.mark.parametrize('compile_options, num_gpus', [ + ('OPTIMIZE', 1), ('OPTIMIZE MAC_METAL', 0), +]) +def test_mips_mps_requires_metal_faiss( + monkeypatch: pytest.MonkeyPatch, + compile_options: str, + num_gpus: int, +) -> None: + import faiss + + monkeypatch.setattr(faiss, 'get_compile_options', lambda: compile_options) + monkeypatch.setattr(faiss, 'get_num_gpus', lambda: num_gpus) + with pytest.raises(RuntimeError, match='Metal-enabled Faiss'): + MIPSFaiss(torch.ones(2, 4, device='mps')) + + +@pytest.mark.skipif(not torch.backends.mps.is_available(), + reason='MPS unavailable') +@pytest.mark.parametrize('k', [1, MAX_K]) +def test_mips_metal_large_topk(k: int) -> None: + rhs = torch.arange(MAX_K + 3, device='mps', dtype=torch.float32).view(-1, 1) + lhs = torch.tensor([[1.0], [2.0]], device='mps') + score, index = MIPSFaiss(rhs).search(lhs, k) + expected_index = torch.arange(rhs.size(0) - 1, rhs.size(0) - k - 1, -1, + device='mps').expand(2, -1) + torch.testing.assert_close(index, expected_index) + torch.testing.assert_close(score, expected_index * lhs) + + +@pytest.mark.skipif(not torch.backends.mps.is_available(), + reason='MPS unavailable') +def test_mips_metal_all_keys_excluded() -> None: + rhs = torch.eye(4, device='mps') + lhs = torch.ones(1, 4, device='mps', requires_grad=True) + exclude = (torch.zeros(4, device='mps', dtype=torch.long), + torch.arange(4, device='mps')) + score, index = MIPSFaiss(rhs).search(lhs, 2, exclude) + assert score.device == index.device == lhs.device + assert torch.isneginf(score).all() + score.masked_fill(torch.isinf(score), 0).sum().backward() + torch.testing.assert_close(lhs.grad, torch.zeros_like(lhs)) + + +@pytest.mark.parametrize('with_exclude', [False, True]) +def test_macos_cpu_retrieval_does_not_require_faiss( + monkeypatch: pytest.MonkeyPatch, + with_exclude: bool, +) -> None: + monkeypatch.setattr(sys.modules[MIPSFaiss.__module__], 'sys', + SimpleNamespace(platform='darwin')) + # Importing Faiss must fail if the CPU path attempts to load it. + monkeypatch.setitem(sys.modules, 'faiss', None) + rhs = torch.eye(4, dtype=torch.float64) + lhs = torch.tensor([[1., 2., 3., 4.]], dtype=torch.float64, + requires_grad=True) + expected_lhs = lhs.detach().clone().requires_grad_() + exclude = (torch.tensor([0]), torch.tensor([3])) if with_exclude else None + index = MIPSFaiss(rhs) + assert isinstance(index._cpu_index, MIPSBruteForce) + score, ids = index.search(lhs, 2, exclude) + expected_score, expected_ids = MIPSBruteForce(rhs).search( + expected_lhs, 2, exclude) + torch.testing.assert_close(score, expected_score) + torch.testing.assert_close(ids, expected_ids) + score.sum().backward() + expected_score.sum().backward() + torch.testing.assert_close(lhs.grad, expected_lhs.grad) + + +@pytest.mark.skipif(sys.platform != 'darwin', reason='macOS dispatch') +@pytest.mark.parametrize('first_device', ['cpu', 'mps']) +def test_macos_cpu_and_metal_retrieval_in_one_process( + first_device: str, +) -> None: + if first_device == 'mps' and not torch.backends.mps.is_available(): + pytest.skip('MPS unavailable') + # A subprocess turns a native OpenMP abort/segfault into a test failure. + script = ''' +import sys +import torch +from kumo_core.modeling.retrieval.mips import MIPSBruteForce, MIPSFaiss + +assert 'faiss' not in sys.modules +torch.manual_seed(123) +devices = ['cpu'] +if torch.backends.mps.is_available(): + devices = (['cpu', 'mps', 'cpu'] if sys.argv[1] == 'cpu' + else ['mps', 'cpu', 'mps']) +for device in devices: + rhs = torch.randn(1024, 32, device=device) + lhs = torch.randn(128, 32, device=device, requires_grad=True) + index = MIPSFaiss(rhs) + scores, ids = index.search(lhs, 5) + expected_scores, expected_ids = MIPSBruteForce(rhs).search(lhs, 5) + torch.testing.assert_close(scores, expected_scores, rtol=1e-4, atol=1e-4) + torch.testing.assert_close(ids, expected_ids) + scores.sum().backward() + assert torch.isfinite(lhs.grad).all() + layer = torch.nn.Linear(32, 16) + layer(torch.randn(512, 32)).sum().backward() + if device == 'cpu' and len(devices) == 1: + assert 'faiss' not in sys.modules +print('CPU and Metal retrieval passed') +''' + env = { + key: value for key, value in os.environ.items() + if not key.startswith(('OMP_', 'KMP_', 'MKL_', 'VECLIB_')) + } + result = subprocess.run([sys.executable, '-c', script, first_device], env=env, + capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/packages/kumo-engine/pyproject.toml b/packages/kumo-engine/pyproject.toml index 03dd919fe..6a0b51c0d 100644 --- a/packages/kumo-engine/pyproject.toml +++ b/packages/kumo-engine/pyproject.toml @@ -37,8 +37,8 @@ test=[ "pytest-mock", "torch>=2.9,<2.13", ] -# kumo-core (Torch/PyG/DiskGraph/faiss-gpu included, via the `gpu` extra) is -# a required dependency of this package now, not an extra -- see setup.py. +# kumo-core (Torch/PyG/DiskGraph included) and platform-selected Faiss are +# required dependencies of this package, not extras -- see setup.py. # Unlike kumo-sdk, where the equivalent `core` extra is optional because # CloudBackend is the default path, this package has no non-training use # case. These per-vendor extras only need to add their own plugin package on diff --git a/packages/kumo-engine/scripts/build_wheel.sh b/packages/kumo-engine/scripts/build_wheel.sh index b5ccc1bf0..6dc60a854 100755 --- a/packages/kumo-engine/scripts/build_wheel.sh +++ b/packages/kumo-engine/scripts/build_wheel.sh @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # # Build the one published `kumo-engine` wheel: the local training closure plus -# every provider package, merged into a single linux x86_64 wheel. +# every provider package, merged into a single native-platform wheel. # # pip install kumo-engine # engine + provider *code* # pip install 'kumo-engine[snowflake]' # ... + snowflake-connector, ... @@ -24,9 +24,8 @@ # Wheel tag: DiskGraph is the only native component and is a stable-ABI # (abi3) extension, so one wheel serves every CPython at or above its floor, # currently 3.11. The merger intersects every component Requires-Python -# constraint for the resulting wheel. The platform tag is manylinux_, -# backed by a check that the bundled binaries really need no newer glibc -# (see check_glibc_floor). +# constraint for the resulting wheel. Linux uses manylinux_, +# backed by check_glibc_floor. macOS inherits DiskGraph's native platform tag. # # Bundled first-party and native components: # @@ -62,13 +61,24 @@ RELEASE=0 # Build DiskGraph through cargo-zigbuild so the extension and the ingest binary # link against a manylinux2014 glibc rather than this host's. A native build # links against the build host's glibc and will not load on the runtime. -USE_ZIG="${USE_ZIG:-1}" +HOST_OS="$(uname -s)" +if [ "${HOST_OS}" = "Darwin" ]; then + USE_ZIG="${USE_ZIG:-0}" + # Matches the minimum of the macOS ARM64 Faiss dependency. Both the + # extension and standalone ingest must use the same deployment target. + export MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET:-14.0}" +else + USE_ZIG="${USE_ZIG:-1}" +fi # Highest glibc the DiskGraph binaries may require, and the manylinux tag the # wheel claims. Container Runtime is Ubuntu-based and provides 2.35, so 2.34 # leaves headroom while still catching a binary accidentally linked against a # modern build host. See check_glibc_floor. MAX_GLIBC="${MAX_GLIBC:-2.34}" -PLATFORM_TAG="${PLATFORM_TAG:-manylinux_${MAX_GLIBC//./_}_x86_64}" +PLATFORM_TAG="${PLATFORM_TAG:-}" +if [ "${HOST_OS}" != "Darwin" ]; then + : "${PLATFORM_TAG:=manylinux_${MAX_GLIBC//./_}_x86_64}" +fi VERIFY_IMAGE="${VERIFY_IMAGE:-}" THIRD_PARTY_NOTICE="${THIRD_PARTY_NOTICE:-}" GENERATE_THIRD_PARTY_NOTICE=0 @@ -87,11 +97,12 @@ Options: --release Stamp exactly the VERSION file's version, for publishing. Without it: +. --version VERSION Override the version entirely (PEP 440). - --platform-tag TAG Wheel platform tag. [manylinux_2_34_x86_64] + --platform-tag TAG Wheel platform tag. [manylinux_2_34_x86_64 on Linux; + native DiskGraph tag on macOS] --kumo-repo PATH kumo monorepo checkout. [this repo] --no-zig Build DiskGraph natively instead of through - cargo-zigbuild. Only safe when this host's glibc is no - newer than the runtime's. + cargo-zigbuild. Default on macOS. On Linux, the host's + glibc must be no newer than the runtime's. --verify-image IMAGE After building, install the wheel into IMAGE with Docker and import the closure as a smoke test. --offline-report Print the requirements pip must still resolve at @@ -100,6 +111,8 @@ Options: -h, --help This message. Environment: + MACOSX_DEPLOYMENT_TARGET Minimum macOS version for both native binaries. + [14.0] MAX_GLIBC Highest glibc the DiskGraph binaries may require. [2.34; Container Runtime provides 2.35] EOF @@ -128,6 +141,10 @@ DISKGRAPH_SOURCE="${KUMO_REPO}/packages/diskgraph" log() { echo "[wheel] $*" >&2; } die() { echo "[wheel] ERROR: $*" >&2; exit 1; } +if [ "${HOST_OS}" = "Darwin" ] && [ "${USE_ZIG}" != "0" ]; then + die "macOS wheels require a native DiskGraph build (USE_ZIG=0)" +fi + # The build host's interpreter only drives pip and the merge. Every pure-Python # component is built with `pip wheel`; DiskGraph's wheel tag comes from its # abi3 build config, not from this interpreter. @@ -253,6 +270,20 @@ env "${ZIG_ENV[@]}" "${PYTHON}" -m pip wheel --no-deps -q \ -w "${WHEELS}" "${DISKGRAPH_SOURCE}" >&2 record 'diskgraph-*.whl' diskgraph-native "in-tree source ${DISKGRAPH_SOURCE}" +if [ "${HOST_OS}" = "Darwin" ]; then + DG_WHEEL="$(ls -1 "${WHEELS}"/diskgraph-*.whl | sort | tail -1)" + DG_PLATFORM="$(basename "${DG_WHEEL}" .whl)" + DG_PLATFORM="${DG_PLATFORM##*-}" + case "${DG_PLATFORM}" in + macosx_*_"$(uname -m)") ;; + *) die "expected a native macOS DiskGraph wheel, got ${DG_WHEEL}" ;; + esac + if [ -n "${PLATFORM_TAG}" ] && [ "${PLATFORM_TAG}" != "${DG_PLATFORM}" ]; then + die "platform tag ${PLATFORM_TAG} does not match DiskGraph's ${DG_PLATFORM}" + fi + PLATFORM_TAG="${DG_PLATFORM}" +fi + if [ "${USE_ZIG}" = "1" ]; then DG_WHEEL="$(ls -1 "${WHEELS}"/diskgraph-*.whl | sort | tail -1)" case "$(basename "${DG_WHEEL}")" in @@ -304,7 +335,9 @@ check_glibc_floor() { fi log "DiskGraph glibc floor: ${worst} (ceiling ${MAX_GLIBC})" } -check_glibc_floor +if [ "${HOST_OS}" != "Darwin" ]; then + check_glibc_floor +fi # -------------------------------------------------------------------------- # 3) Generate the legal payload and combined license expression. @@ -349,7 +382,7 @@ fi WHEEL_PATH="$("${PYTHON}" "${SCRIPT_DIR}/merge_wheels.py" "${WHEELS}" \ "${COMPONENTS}" "${OUTPUT_DIR}" "${merge_args[@]}")" -SIZE_MIB="$(( $(stat -c %s "${WHEEL_PATH}") / 1024 / 1024 ))" +SIZE_MIB="$("${PYTHON}" -c 'import os, sys; print(os.path.getsize(sys.argv[1]) // 1024**2)' "${WHEEL_PATH}")" log "wrote ${WHEEL_PATH} (${SIZE_MIB} MiB)" # -------------------------------------------------------------------------- diff --git a/packages/kumo-engine/scripts/merge_wheels.py b/packages/kumo-engine/scripts/merge_wheels.py index dd9b0b192..3fa1110fd 100644 --- a/packages/kumo-engine/scripts/merge_wheels.py +++ b/packages/kumo-engine/scripts/merge_wheels.py @@ -419,6 +419,17 @@ def main(): destination.chmod(destination.stat().st_mode | 0o111) log(f"merged {component['name']} {component['version']}") + # A development bundle adds its commit to the component's stamped version. + # Keep the installed module consistent with the assembled wheel metadata. + version_module = staging / 'kumo/engine/_version.py' + if version_module.is_file(): + source, count = re.subn(r'(?m)^__version__\s*=.*$', + f'__version__ = {version!r}', + version_module.read_text(encoding='utf-8')) + if count != 1: + fail('expected one stamped kumo.engine version') + version_module.write_text(source, encoding='utf-8') + dist_info = staging / f"{name.replace('-', '_')}-{version}.dist-info" dist_info.mkdir(parents=True) if entry_points.sections(): diff --git a/packages/kumo-engine/setup.py b/packages/kumo-engine/setup.py index 125b39632..2c174c3c3 100644 --- a/packages/kumo-engine/setup.py +++ b/packages/kumo-engine/setup.py @@ -39,12 +39,11 @@ # Unlike kumo-sdk (where this is an optional extra behind # CloudBackend), this package has no non-training use case, so # kumo-core -- and the Torch/PyG/DiskGraph stack it pulls in -- is a - # plain, required dependency here. Pull in the `faiss-gpu` extra - # (the official faiss-gpu package) unconditionally: faiss-gpu is a - # functional superset of faiss-cpu (its CPU index classes work with - # no GPU present), so this avoids requiring every kumo-engine user - # to pick a faiss-cpu/faiss-gpu extra, and sidesteps faiss-cpu/ - # faiss-gpu both being importable as `faiss` and corrupting each - # other's install if ever combined. - "kumo-core[faiss-gpu]>=0.1.0.dev1", + # plain, required dependency here. + "kumo-core>=0.1.0.dev1", + # Select one Faiss distribution: both provide the `faiss` module. + # Keep platform markers on these external dependencies so wheel + # merging preserves them when the bundled core reference is removed. + 'faiss-gpu==1.14.3; sys_platform == "linux"', + 'faiss-cpu==1.14.3; sys_platform == "darwin"', ], ) diff --git a/test/scripts/test_engine_wheel.py b/test/scripts/test_engine_wheel.py index c6be130ec..3345a6de1 100644 --- a/test/scripts/test_engine_wheel.py +++ b/test/scripts/test_engine_wheel.py @@ -4,8 +4,10 @@ import json import os +import runpy import subprocess import sys +import types import zipfile from email.parser import BytesParser from pathlib import Path @@ -202,6 +204,61 @@ def _read_wheel(tmp_path): return metadata, manifest, names +def test_merged_module_reports_bundle_version(tmp_path, monkeypatch): + monkeypatch.setitem(COMPONENTS, 'kumo-engine', { + **COMPONENTS['kumo-engine'], + 'files': {'kumo/engine/_version.py': '__version__ = "0.1"\n'}, + }) + result = _merge(tmp_path, list(COMPONENTS)) + assert result.returncode == 0, result.stderr + with ZipFile(next((tmp_path / 'dist').glob('*.whl'))) as archive: + assert archive.read('kumo/engine/_version.py') == ( + b"__version__ = '1.0'\n") + + +@pytest.mark.parametrize(('platform', 'expected'), [ + ('linux', {'faiss-gpu'}), + ('darwin', {'faiss-cpu'}), + ('win32', set()), +]) +def test_engine_faiss_platform_selection_survives_merge( + tmp_path, monkeypatch, platform, expected): + from packaging.requirements import Requirement + + setup_kwargs = {} + monkeypatch.setitem(sys.modules, 'setuptools', types.SimpleNamespace( + setup=lambda **kwargs: setup_kwargs.update(kwargs))) + runpy.run_path(str(REPO_ROOT / 'packages/kumo-engine/setup.py')) + engine = COMPONENTS['kumo-engine'] + provider_requirements = [ + requirement for requirement in engine['requires'] + if 'extra ==' in requirement + ] + monkeypatch.setitem(COMPONENTS, 'kumo-engine', { + **engine, + 'requires': setup_kwargs['install_requires'] + provider_requirements, + }) + result = _merge(tmp_path, list(COMPONENTS)) + assert result.returncode == 0, result.stderr + metadata, manifest, _ = _read_wheel(tmp_path) + faiss = [ + Requirement(value) for value in metadata.get_all('Requires-Dist') + if value.startswith('faiss-') + ] + assert {requirement.name for requirement in faiss} == { + 'faiss-cpu', 'faiss-gpu' + } + assert all(str(requirement.specifier) == '==1.14.3' + and requirement.marker is not None for requirement in faiss) + selected = { + requirement.name for requirement in faiss + if requirement.marker.evaluate({'sys_platform': platform, 'extra': ''}) + } + assert selected == expected + assert {str(requirement) for requirement in faiss} <= set( + manifest['requires']) + + @pytest.mark.parametrize('providers', [ ['kumo-databricks', 'kumo-s3', 'kumo-snowflake'], ['kumo-databricks'], @@ -482,8 +539,9 @@ def run(): ([], '2.5.0+unknown'), # VERSION + commit (not a git repo here) (['--release'], '2.5.0'), ]) +@pytest.mark.parametrize('host_os', ['Linux', 'Darwin']) def test_wheel_uses_torch_fallbacks_without_native_extensions( - tmp_path, args, version): + tmp_path, args, version, host_os): """Run the real assembly script with local stand-ins for component builds.""" repo = tmp_path / 'repo' for name in CORE_COMPONENTS: @@ -514,7 +572,12 @@ def test_wheel_uses_torch_fallbacks_without_native_extensions( directory = Path(args[args.index('-w' if args[2] == 'wheel' else '-d') + 1]) for source in (args[-1:] if args[2] == 'wheel' else args[-2:]): name = Path(source).name.split('==')[0].replace('-', '_') - with ZipFile(directory / (name + '-1-py3-none-any.whl'), 'w') as archive: + if name == 'diskgraph' and os.environ['TEST_HOST_OS'] == 'Darwin': + assert os.environ['DISKGRAPH_BUILD_ZIG'] == '0' + assert os.environ['MACOSX_DEPLOYMENT_TARGET'] == '14.0' + tag = ('cp311-abi3-macosx_14_0_arm64' if name == 'diskgraph' + and os.environ['TEST_HOST_OS'] == 'Darwin' else 'py3-none-any') + with ZipFile(directory / (name + '-1-' + tag + '.whl'), 'w') as archive: metadata = 'Metadata-Version: 2.1\\nName: ' + name + '\\nVersion: 1\\n' if name == 'kumo_engine': metadata += ('Requires-Python: >=3.11\\nProvides-Extra: text\\n' @@ -532,6 +595,11 @@ def test_wheel_uses_torch_fallbacks_without_native_extensions( archive.writestr(name + '/__init__.py', '') ''') python.chmod(0o755) + uname = tmp_path / 'uname' + arch = 'arm64' if host_os == 'Darwin' else 'x86_64' + uname.write_text(f'#!/bin/sh\ncase "$1" in\n' + f'-s) echo {host_os} ;;\n-m) echo {arch} ;;\nesac\n') + uname.chmod(0o755) for command in ['cargo', 'curl']: executable = tmp_path / command executable.write_text('#!/bin/bash\nexit 97\n') @@ -539,10 +607,13 @@ def test_wheel_uses_torch_fallbacks_without_native_extensions( output = tmp_path / 'dist' result = subprocess.run( ['bash', - str(SCRIPT), '--no-zig', *args, '--output-dir', + str(SCRIPT), *(['--no-zig'] if host_os == 'Linux' else []), + *args, '--output-dir', str(output)], env=dict(os.environ, KUMO_REPO=str(repo), PYTHON=str(python), BUILD_CALLS=str(calls), TMPDIR=str(tmp_path), + TEST_HOST_OS=host_os, USE_ZIG='', PLATFORM_TAG='', + MACOSX_DEPLOYMENT_TARGET='', THIRD_PARTY_NOTICE=str(repo / 'THIRD-PARTY.txt'), PATH=f'{tmp_path}:{os.environ["PATH"]}'), capture_output=True, @@ -554,6 +625,9 @@ def test_wheel_uses_torch_fallbacks_without_native_extensions( if json.loads(line)[2] == 'download' ] assert not downloads + if host_os == 'Darwin': + assert next(output.glob('*.whl')).name.endswith( + '-cp311-abi3-macosx_14_0_arm64.whl') with ZipFile(next(output.glob('*.whl'))) as archive: assert not any(name.startswith('torch_geometric/') for name in archive.namelist()) @@ -576,5 +650,7 @@ def test_wheel_uses_torch_fallbacks_without_native_extensions( assert 'Requires-Dist: kumo-glove-6b-50d; extra == "text"' in metadata assert 'Requires-Dist: pyg-lib' not in metadata assert 'Requires-Dist: torch-scatter' not in metadata + expected_tag = ('cp311-abi3-macosx_14_0_arm64' + if host_os == 'Darwin' else 'py3-none-any') assert next(output.glob('*.whl')).name == ( - f'kumo_engine-{version}-py3-none-any.whl') + f'kumo_engine-{version}-{expected_tag}.whl') diff --git a/uv.lock b/uv.lock index bc34f70ea..0fbdc6e50 100644 --- a/uv.lock +++ b/uv.lock @@ -1591,21 +1591,23 @@ wheels = [ [[package]] name = "faiss-cpu" -version = "1.14.2" +version = "1.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "packaging" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/8b/5d2cd7c9fd60bc4d1ca6e9f5e8b0eb57254a7b301daaa12d5853fdf48afe/faiss_cpu-1.14.2-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a20011b8a97318e6d5e29143a773277ca71f1c33140024aeec91f05ad56cdd04", size = 4621203, upload-time = "2026-05-22T19:58:27.415Z" }, - { url = "https://files.pythonhosted.org/packages/84/b1/05876aa7ceafd67a8c53667f574c0354e50ebadfdaf0632d7d9f5c80fffe/faiss_cpu-1.14.2-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:6ba528b5803fe5206bbb38e6ea2c537de0d1482a47c5765982af4e4a48c135e2", size = 6681426, upload-time = "2026-05-22T19:58:29.193Z" }, - { url = "https://files.pythonhosted.org/packages/35/b4/d130a1908ad548671cd5ee9a1b537c7d3753cfdf0b2b134f3c10ccc6b537/faiss_cpu-1.14.2-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b340c33212351f04d4f167cb7b22c9e756705a634d3b7b58987651f3ba1f6217", size = 9592827, upload-time = "2026-05-22T19:58:30.772Z" }, - { url = "https://files.pythonhosted.org/packages/7f/66/108f7075591b84852a6b285a81922e773c1c6d3b2c45da8ec4822f1bfab4/faiss_cpu-1.14.2-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ec4784d9a14973f2eead3a3480e6d14e74253b234e2c12717d4319f21dfadfcd", size = 18234781, upload-time = "2026-05-22T19:58:33.268Z" }, - { url = "https://files.pythonhosted.org/packages/56/b9/a4d076acd9eeaf6fb6f5a29de1a7a0a34de411d0fbe3d8f8c91abb9f97e0/faiss_cpu-1.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:a9d81699ff3ae6e25244efda48c52e406963bbce16d5754073725c14620885c6", size = 16111239, upload-time = "2026-05-22T19:58:38.889Z" }, - { url = "https://files.pythonhosted.org/packages/f2/42/af4df8012e0ed817df2c4094a816b538d99bfbbcbf567cd9f5f36d5fb9b4/faiss_cpu-1.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:292124ba3d2bbeed920ef8519451cfba0f139b77db1607170cffbe68dc72e604", size = 16115701, upload-time = "2026-05-22T19:58:41.85Z" }, - { url = "https://files.pythonhosted.org/packages/60/04/555551ceec73248e76b758014a6fba702349824270be27d0e5db1f740feb/faiss_cpu-1.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:d81857f1fcf6c5dc76f75b35112158f731aeb7804ca971c51d74edf6e9b1b8c7", size = 16116212, upload-time = "2026-05-22T19:58:45.154Z" }, - { url = "https://files.pythonhosted.org/packages/9e/59/ff73cb48bd1a98a355ff5a6a2018f3cfe4980ec2196b4938bf8af4fb828b/faiss_cpu-1.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:b736813d853d36693f7b5affdc8788adfa73efa2f45ac6493c8ce2cbb2584f19", size = 16394291, upload-time = "2026-05-22T19:58:48.062Z" }, + { url = "https://files.pythonhosted.org/packages/83/b0/48c083d01b7b68c463c1d56507147a9d733f791e1c469a77215a872a9fb5/faiss_cpu-1.14.3-cp310-abi3-macosx_14_0_arm64.whl", hash = "sha256:a9369863290a3f0e033757e4c10577b6ef7431f1cede394dabd0a137e4e2ed45", size = 4768290, upload-time = "2026-06-13T02:19:03.427Z" }, + { url = "https://files.pythonhosted.org/packages/ab/34/6b04ef5bae3eada6b5a9457d7875cce041c040d53c890815cbd1e9821c65/faiss_cpu-1.14.3-cp310-abi3-macosx_15_0_x86_64.whl", hash = "sha256:f9d0e84d909194f63f027bbd3c1e35e905e48c9345c2db6e6f24da09a6bc5906", size = 6925734, upload-time = "2026-06-13T02:19:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/b451af4b3c6dd18749ecfb58ccb68503b77e49c9aa4a89d950e9d521e058/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d734cfa9ac90b6a5dfed3a27cb706d05f22824703dafc3969b4e2071877a31c", size = 9661210, upload-time = "2026-06-13T02:19:06.99Z" }, + { url = "https://files.pythonhosted.org/packages/a0/ed/57335bc18c9e18677587bec9bf070c675b29c8e683e13f4def0440731ca0/faiss_cpu-1.14.3-cp310-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8780b526c06e57aad90a8c4655dfba2ff1b3195bd600ff4499752fc45159c9fc", size = 18506292, upload-time = "2026-06-13T02:19:09.621Z" }, + { url = "https://files.pythonhosted.org/packages/e7/d3/c6ca8c44a63b909e78aa8a69e14501c79c89613e62261d58455ea603d710/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:b28ba083e8c02f2c9be03783402537fd3f00d27e68799c44bf88931500ee12ec", size = 11238800, upload-time = "2026-06-13T02:19:12.821Z" }, + { url = "https://files.pythonhosted.org/packages/93/5f/b405692913a301251749cb175cb3f564ed257fdaa80a22c9a36444d0095d/faiss_cpu-1.14.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:cdcb90850cb4b7c27d270839b37bcc0dc8d1fb6a62d1e13053e51ac95061ca25", size = 19237637, upload-time = "2026-06-13T02:19:15.531Z" }, + { url = "https://files.pythonhosted.org/packages/78/6c/bbf670ad727502102fa90c52f9b47c73753f0adda880d30427ac1d48b37b/faiss_cpu-1.14.3-cp311-cp311-win_amd64.whl", hash = "sha256:f278003d00c30c90cf118b98428bb96f73eba4b9dadac6993c788966d2e983bd", size = 16159350, upload-time = "2026-06-13T02:19:20.601Z" }, + { url = "https://files.pythonhosted.org/packages/59/aa/bfa53255a6aa6e79b2471bb5af895f662feb02b612e1e0f9efc3d893286e/faiss_cpu-1.14.3-cp312-cp312-win_amd64.whl", hash = "sha256:0a5eb27184123c7ac1060c6b862978eabf0e30c1369ccf8bdb1497d35c06ad3d", size = 16164699, upload-time = "2026-06-13T02:19:22.969Z" }, + { url = "https://files.pythonhosted.org/packages/1d/c1/2fb14f58ff74d7a7d6fd13084c016d9b144ce0bcdf77f6526cc2e4828278/faiss_cpu-1.14.3-cp313-cp313-win_amd64.whl", hash = "sha256:ea2340f675db59af8db6da4535b541ce6948d68a008989c656c292c7b0c77127", size = 16162836, upload-time = "2026-06-13T02:19:25.529Z" }, + { url = "https://files.pythonhosted.org/packages/fd/aa/100d3b706b8e2aa561e2673ed7d07c4c7c26f92d938a88d193dd54f90c45/faiss_cpu-1.14.3-cp314-cp314-win_amd64.whl", hash = "sha256:54b0ea832be5bce75ef9abf1297fe4613d760bf7cfe7a963da040cb0314e8f58", size = 16444608, upload-time = "2026-06-13T02:19:27.992Z" }, ] [[package]] @@ -3199,7 +3201,7 @@ text = [ [package.metadata] requires-dist = [ { name = "diskgraph", editable = "packages/diskgraph" }, - { name = "faiss-cpu", marker = "extra == 'faiss-cpu'", specifier = "==1.14.2" }, + { name = "faiss-cpu", marker = "extra == 'faiss-cpu'", specifier = "==1.14.3" }, { name = "faiss-gpu", marker = "extra == 'dev'", specifier = "==1.14.3" }, { name = "faiss-gpu", marker = "extra == 'faiss-gpu'", specifier = "==1.14.3" }, { name = "faiss-gpu", marker = "extra == 'test'", specifier = "==1.14.3" }, @@ -3272,9 +3274,11 @@ provides-extras = ["test"] name = "kumo-engine" source = { editable = "packages/kumo-engine" } dependencies = [ + { name = "faiss-cpu", marker = "sys_platform == 'darwin' or (extra == 'extra-4-kumo-cpu' and extra == 'extra-4-kumo-cu130') or (extra == 'extra-4-kumo-cpu' and extra == 'group-4-kumo-public-tests') or (extra == 'extra-4-kumo-cu130' and extra == 'group-4-kumo-public-tests')" }, + { name = "faiss-gpu", marker = "sys_platform == 'linux' or (extra == 'extra-4-kumo-cpu' and extra == 'extra-4-kumo-cu130') or (extra == 'extra-4-kumo-cpu' and extra == 'group-4-kumo-public-tests') or (extra == 'extra-4-kumo-cu130' and extra == 'group-4-kumo-public-tests')" }, { name = "filelock" }, { name = "kumo-api" }, - { name = "kumo-core", extra = ["faiss-gpu"] }, + { name = "kumo-core" }, { name = "pandas" }, { name = "pyarrow" }, { name = "pydantic" }, @@ -3319,10 +3323,12 @@ requires-dist = [ { name = "boto3", marker = "extra == 'test'", specifier = ">=1.34,<2" }, { name = "botocore", marker = "extra == 'test'", specifier = ">=1.34,<2" }, { name = "diskgraph", marker = "extra == 'test'", editable = "packages/diskgraph" }, + { name = "faiss-cpu", marker = "sys_platform == 'darwin'", specifier = "==1.14.3" }, + { name = "faiss-gpu", marker = "sys_platform == 'linux'", specifier = "==1.14.3" }, { name = "filelock", specifier = ">=3.0.0" }, { name = "kumo-api", editable = "packages/kumo-api" }, { name = "kumo-bigquery", marker = "extra == 'bigquery'", editable = "packages/kumo-bigquery" }, - { name = "kumo-core", extras = ["faiss-gpu"], editable = "packages/kumo-core" }, + { name = "kumo-core", editable = "packages/kumo-core" }, { name = "kumo-databricks", marker = "extra == 'databricks'", editable = "packages/kumo-databricks" }, { name = "kumo-gcs", marker = "extra == 'gcs'", editable = "packages/kumo-gcs" }, { name = "kumo-glove-6b-50d", marker = "extra == 'text'", editable = "tools/glove-6b-50d" }, From 4d8a759e2bf7dbcc5018cb48984444ab4339026a Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 13:38:03 -0700 Subject: [PATCH 02/39] Expose per-trainer accelerator selection --- .../kumo-core/src/kumo_core/prediction.py | 16 +++- .../src/kumo_core/training/runtime.py | 8 +- .../tests/test_local_training_runtime.py | 8 ++ packages/kumo-core/tests/test_prediction.py | 29 ++++++- packages/kumo-engine/README.md | 18 +++++ .../kumo/engine/trainer/_runtime.py | 5 +- .../kumo/engine/trainer/trainer.py | 30 +++++-- packages/kumo-engine/test/test_accelerator.py | 81 +++++++++++++++++++ .../test/test_prediction_completion.py | 2 +- .../test/test_prediction_workers.py | 31 +++++-- .../kumo-engine/test/test_trainer_registry.py | 13 ++- .../test/test_training_explanations.py | 8 +- 12 files changed, 222 insertions(+), 27 deletions(-) create mode 100644 packages/kumo-engine/test/test_accelerator.py diff --git a/packages/kumo-core/src/kumo_core/prediction.py b/packages/kumo-core/src/kumo_core/prediction.py index 885e3ae89..e4dcfaec2 100644 --- a/packages/kumo-core/src/kumo_core/prediction.py +++ b/packages/kumo-core/src/kumo_core/prediction.py @@ -462,7 +462,10 @@ def _run_prediction_workers( return results -def _prediction_worker_devices(num_workers: int) -> Tuple[Optional[int], ...]: +def _prediction_worker_devices( + num_workers: int, + accelerator: Optional[str] = None, +) -> Tuple[Optional[int], ...]: r"""Assign concurrent scorers to distinct visible CUDA devices. ``None`` leaves Lightning's automatic non-CUDA selection in charge. A @@ -471,7 +474,9 @@ def _prediction_worker_devices(num_workers: int) -> Tuple[Optional[int], ...]: """ if num_workers == 1: return (None, ) - if os.environ.get('KUMO_ACCELERATOR', 'auto').lower() == 'cpu': + accelerator = (accelerator if accelerator is not None else + os.environ.get('KUMO_ACCELERATOR', 'auto')).lower() + if accelerator in {'cpu', 'mps'}: return (None, ) * num_workers import torch @@ -507,6 +512,7 @@ class _PredictionWorkerConfig(NamedTuple): forecast_options: Optional[Dict[str, int]] embedding_models: Optional[EmbeddingModelStore] = None ignore_time_col: bool = False + accelerator: Optional[str] = None def _score_prediction_shard( @@ -535,6 +541,8 @@ def _score_prediction_shard( **(config.forecast_options or {}), ) load_options = {} + if config.accelerator is not None: + load_options['accelerator'] = config.accelerator device = config.devices[worker_id] if device is not None: load_options['devices'] = [device] @@ -606,6 +614,7 @@ def score_prediction( forecast_options: Mapping[str, int] | None = None, embedding_models: EmbeddingModelStore | None = None, encoders: Any = None, + accelerator: str | None = None, ) -> Any: """Score partitions and publish bounded metadata using the hosted protocol. @@ -658,7 +667,7 @@ def score_prediction( attempt_id = uuid4().hex config = _PredictionWorkerConfig( worker_count=worker_count, - devices=_prediction_worker_devices(worker_count), + devices=_prediction_worker_devices(worker_count, accelerator), manifest_path=materialized.manifest_path, bundle_dir=materialized.bundle_dir, model_dir=model_dir, @@ -678,6 +687,7 @@ def score_prediction( forecast_options=(dict(forecast_options) if forecast_options is not None else None), embedding_models=embedding_models, + accelerator=accelerator, ) if worker_count == 1: references = [ diff --git a/packages/kumo-core/src/kumo_core/training/runtime.py b/packages/kumo-core/src/kumo_core/training/runtime.py index 096584f72..8b1489b78 100644 --- a/packages/kumo-core/src/kumo_core/training/runtime.py +++ b/packages/kumo-core/src/kumo_core/training/runtime.py @@ -143,11 +143,13 @@ def __init__( resolved_task: Any, run_name: Optional[str] = None, embedding_models: Optional[EmbeddingModelStore] = None, + accelerator: Optional[str] = None, ) -> None: self.home_dir = str(Path(home_dir)) self.resolved_task = resolved_task self.run_name = run_name self.embedding_models = embedding_models + self.accelerator = accelerator self.artifact_runtime = LocalTrainingArtifactRuntime(self.home_dir) self.diskgraph_runtime = LocalDiskgraphTrainingRuntime() self.explain_runtime = LocalExplainRuntime() @@ -183,6 +185,8 @@ def load_automl_config(self, job_config: Any) -> Any: def trainer_runtime_kwargs(self) -> Dict[str, Any]: return { + **({'accelerator': self.accelerator} + if self.accelerator is not None else {}), 'artifact_runtime': self.artifact_runtime, 'diskgraph_runtime': self.diskgraph_runtime, 'explain_runtime': self.explain_runtime, @@ -202,6 +206,7 @@ def local_training_bindings( training_input: Any, run_name: Optional[str] = None, embedding_models: Optional[EmbeddingModelStore] = None, + accelerator: Optional[str] = None, ) -> Any: r"""Binds one resolved training input to this process. @@ -218,7 +223,8 @@ def local_training_bindings( runtime=LocalAutoTrainingRuntime(home_dir=training_input.home_dir, resolved_task=training_input.task, run_name=run_name, - embedding_models=embedding_models), + embedding_models=embedding_models, + accelerator=accelerator), instrumentation=NoOpTrainingInstrumentation(), resolve_vital_artifacts=resolve_vital_artifact_sources, # The vital bundle lands beside the run. A platform backend would then diff --git a/packages/kumo-core/tests/test_local_training_runtime.py b/packages/kumo-core/tests/test_local_training_runtime.py index c8f60e0b5..0bae4abde 100644 --- a/packages/kumo-core/tests/test_local_training_runtime.py +++ b/packages/kumo-core/tests/test_local_training_runtime.py @@ -94,3 +94,11 @@ def test_default_bindings_supply_shared_runtimes(tmp_path): kwargs = bindings.runtime.trainer_runtime_kwargs() assert kwargs['artifact_runtime'] is bindings.runtime.artifact_runtime assert kwargs['embedding_models'] is store + assert 'accelerator' not in kwargs + + +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) +def test_bindings_pass_instance_accelerator(tmp_path, accelerator): + training_input = SimpleNamespace(home_dir=str(tmp_path), task=object()) + bindings = local_training_bindings(training_input, accelerator=accelerator) + assert bindings.runtime.trainer_runtime_kwargs()['accelerator'] == accelerator diff --git a/packages/kumo-core/tests/test_prediction.py b/packages/kumo-core/tests/test_prediction.py index 32a5677db..14eed4cc2 100644 --- a/packages/kumo-core/tests/test_prediction.py +++ b/packages/kumo-core/tests/test_prediction.py @@ -481,7 +481,8 @@ def thin(spec, **kwargs): "feature_columns"] == [FUTURE_LABELS_COL, AUTOREGRESSION_COL] -def _score_shard_on_device(monkeypatch, tmp_path, devices, worker_id): +def _score_shard_on_device(monkeypatch, tmp_path, devices, worker_id, + accelerator=None): r"""Run one real prediction-worker load into ``create_pl_trainer``.""" from kumo_core.encoders.tensor_frame_encoder import ( HeteroTensorFrameEncoder, @@ -541,7 +542,8 @@ def predict(self, **_): pred_time_col_name=None, output_dir=str(tmp_path / "output"), attempt_id="attempt", graph_snapshot_id="snapshot", source_config_sha256="0" * 64, runtime_config_sha256="1" * 64, - binary_threshold=None, forecast_options=None), + binary_threshold=None, forecast_options=None, + accelerator=accelerator), ) return trainers @@ -557,6 +559,25 @@ def test_prediction_worker_pins_lightning_to_its_device(monkeypatch, tmp_path): for t in trainers] == [("auto", [3])] * 2 +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) +def test_prediction_worker_accelerator_overrides_environment( + monkeypatch, tmp_path, accelerator): + import pytorch_lightning as pl + monkeypatch.setenv('KUMO_ACCELERATOR', 'cpu') + monkeypatch.setattr(pl, 'Trainer', lambda **kwargs: kwargs) + trainers = _score_shard_on_device(monkeypatch, tmp_path, (None,), 0, + accelerator=accelerator) + assert [trainer['accelerator'] for trainer in trainers] == [accelerator] * 2 + + +def test_explicit_cpu_runs_lightning_on_cpu(monkeypatch, tmp_path): + monkeypatch.setenv('KUMO_ACCELERATOR', 'mps') + trainers = _score_shard_on_device(monkeypatch, tmp_path, (None,), 0, + accelerator='cpu') + assert all(trainer.strategy.root_device.type == 'cpu' + for trainer in trainers) + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_prediction_worker_runs_lightning_on_cuda_device(monkeypatch, tmp_path): monkeypatch.delenv("KUMO_ACCELERATOR", raising=False) @@ -592,7 +613,7 @@ def test_score_prediction_metadata_options(monkeypatch, tmp_path, task_type, lambda _: 'runtime-hash') monkeypatch.setattr(artifact_manifest, 'publish_prediction_selection', lambda references, **kwargs: references[0]) - monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda _: + monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda _, accelerator=None: (None, )) monkeypatch.setattr(prediction, '_score_prediction_shard', lambda *, worker_id, config, **_: config) @@ -686,7 +707,7 @@ def load_trainer(*args, **kwargs): lambda references, **kwargs: references[0]) monkeypatch.setattr(prediction, '_publish_scored_shard', lambda *_, **__: None) - monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda _: + monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda _, accelerator=None: (None, )) encoders = object() diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 3847ae5bd..ef6851ca6 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -33,6 +33,24 @@ The package uses `kumoapi` model plans and connector plugin contracts. [Output configurations](https://github.com/NVIDIA/kumo-engine/blob/main/packages/kumo-engine/kumo/engine/trainer/output_config.py) take connector objects directly. +### Training device + +Choose the accelerator when constructing a trainer: + +```python +trainer = Trainer(model_plan, accelerator="cpu") +``` + +Accepted values are `auto` (default), `cpu`, `gpu`, and `mps`, using Lightning's +device selection. The choice applies to training, refits, explanations, and +prediction, and overrides `KUMO_ACCELERATOR` for this instance. An explicitly +requested accelerator must be available on the host. + +Device selection is local to the instance and is not saved in model bundles. +Select it again when loading, for example +`Trainer.load(bundle_path, accelerator="cpu")` or +`Trainer.load_from_connector(registry, path="model", accelerator="cpu")`. + ## Installation Requires Linux x86_64, CPython 3.11 or newer, and glibc 2.34 or newer. diff --git a/packages/kumo-engine/kumo/engine/trainer/_runtime.py b/packages/kumo-engine/kumo/engine/trainer/_runtime.py index 15d372e10..f258d6ac1 100644 --- a/packages/kumo-engine/kumo/engine/trainer/_runtime.py +++ b/packages/kumo-engine/kumo/engine/trainer/_runtime.py @@ -10,8 +10,9 @@ def local_training_bindings(training_input: Any, run_name: Optional[str] = None, - embedding_models: Any = None) -> Any: + embedding_models: Any = None, + accelerator: str = 'auto') -> Any: from kumo_core.training.runtime import local_training_bindings as bind return bind(training_input, run_name=run_name, - embedding_models=embedding_models) + embedding_models=embedding_models, accelerator=accelerator) diff --git a/packages/kumo-engine/kumo/engine/trainer/trainer.py b/packages/kumo-engine/kumo/engine/trainer/trainer.py index e44504897..132e580a5 100644 --- a/packages/kumo-engine/kumo/engine/trainer/trainer.py +++ b/packages/kumo-engine/kumo/engine/trainer/trainer.py @@ -11,7 +11,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone from pathlib import Path -from typing import Dict, Mapping, Optional, cast +from typing import Dict, Literal, Mapping, Optional, cast import pandas as pd from kumo_core.embedding_models import EmbeddingModelStore @@ -208,12 +208,20 @@ class Trainer: between hosted and local training. embedding_models: the word-embedding models for text columns, by name. Defaults to the installed ones (``kumo-engine[text]``). + accelerator: Lightning device selection: ``auto`` (default), ``cpu``, + ``gpu``, or ``mps``. Applies to training and prediction. """ def __init__( self, model_plan: ModelPlan, embedding_models: Optional[EmbeddingModelStore] = None, + *, + accelerator: Literal['cpu', 'gpu', 'mps', 'auto'] = 'auto', ): + if accelerator not in ('cpu', 'gpu', 'mps', 'auto'): + raise ValueError( + "accelerator must be one of 'cpu', 'gpu', 'mps', or 'auto'") + self._accelerator = accelerator self._model_plan = model_plan self._embedding_models = (embedding_models if embedding_models is not None else EmbeddingModelStore()) @@ -419,7 +427,8 @@ def fit( training_input, bindings=local_training_bindings( training_input, run_name=run_id, - embedding_models=self._embedding_models), + embedding_models=self._embedding_models, + accelerator=self._accelerator), options=CallableTrainingOptions(), ) @@ -704,6 +713,7 @@ def predict( forecast_options=forecast_options, embedding_models=self._embedding_models, encoders=encoders, + accelerator=self._accelerator, ) del encoders @@ -718,13 +728,18 @@ def load( cls, bundle_path: str | Path, embedding_models: Optional[EmbeddingModelStore] = None, + *, + accelerator: Literal['cpu', 'gpu', 'mps', 'auto'] = 'auto', ) -> Self: r"""Restore a trainer from an existing local model bundle. Its text columns need the same word-embedding models it was trained with, registered in ``embedding_models`` under any name or installed. + ``accelerator`` selects the device for this instance, defaulting to + ``auto`` independently of the device used to train the model. """ - return cls._load_bundle(Path(bundle_path), embedding_models) + return cls._load_bundle(Path(bundle_path), embedding_models, + accelerator=accelerator) @classmethod def load_from_connector( @@ -734,6 +749,7 @@ def load_from_connector( path: Optional[str] = None, location: Optional[ModelRegistryLocation] = None, embedding_models: Optional[EmbeddingModelStore] = None, + accelerator: Literal['cpu', 'gpu', 'mps', 'auto'] = 'auto', ) -> Self: r"""Fetch a trusted publication, then restore its training state. @@ -745,6 +761,7 @@ def load_from_connector( the bundle does not restore graph storage or source data. Hashes detect corruption, not publisher authentication. Load only models from trusted publishers; checkpoint deserialization happens at predict. + ``accelerator`` selects the device for this instance (default: ``auto``). """ address = ModelOutputConfig(model_registry, path=path, location=location) @@ -755,13 +772,16 @@ def load_from_connector( 'model_registry.sink_backend must implement ModelFetchingSink') sink.fetch_model(cast(SourceConnector, model_registry), str(bundle_dir), path=address.path, location=address.location) - return cls._load_bundle(bundle_dir, embedding_models) + return cls._load_bundle(bundle_dir, embedding_models, + accelerator=accelerator) @classmethod def _load_bundle( cls, bundle_dir: Path, embedding_models: Optional[EmbeddingModelStore] = None, + *, + accelerator: Literal['cpu', 'gpu', 'mps', 'auto'] = 'auto', ) -> Self: bundle_dir = bundle_dir.resolve() paths = read_bundle_artifact_paths(bundle_dir) @@ -771,7 +791,7 @@ def _load_bundle( metrics_path = bundle_dir / 'metrics.json' metrics = (json.loads(metrics_path.read_text( encoding='utf-8')) if metrics_path.is_file() else {}) - trainer = cls(plan, embedding_models) + trainer = cls(plan, embedding_models, accelerator=accelerator) trainer._snapshot_id = spec.snapshot_id trainer._training_query = spec.predictive_query trainer._training_table_generation_plan = from_json_dict( diff --git a/packages/kumo-engine/test/test_accelerator.py b/packages/kumo-engine/test/test_accelerator.py new file mode 100644 index 000000000..a035ff132 --- /dev/null +++ b/packages/kumo-engine/test/test_accelerator.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os + +import pandas as pd +import pytest +from kumoapi.model_plan import ModelPlan + +from kumo.engine import FileConnector, Graph, PredictiveQuery, Table, Trainer +from kumo.engine._scratch import get_scratch_dir + + +@pytest.mark.parametrize('accelerator', ['cuda', 'tpu', '', None, 1]) +def test_invalid_accelerator_fails_at_construction(accelerator): + with pytest.raises(ValueError, match='accelerator must be one of'): + Trainer(ModelPlan(), accelerator=accelerator) + + +def test_accelerator_defaults_to_auto_and_is_instance_local(monkeypatch): + monkeypatch.setenv('KUMO_ACCELERATOR', 'mps') + automatic = Trainer(ModelPlan()) + cpu = Trainer(ModelPlan(), accelerator='cpu') + assert automatic._accelerator == 'auto' + assert cpu._accelerator == 'cpu' + assert os.environ['KUMO_ACCELERATOR'] == 'mps' + + +def test_cpu_fit_refit_reload_and_predict_override_environment( + tmp_path, monkeypatch): + import kumo.engine as kumo + from kumo_core.train.trainer import Trainer as CoreTrainer + + devices = [] + original = CoreTrainer.create_pl_trainer + + def create_pl_trainer(self): + original(self) + devices.append(self.pl_trainer.strategy.root_device.type) + + monkeypatch.setattr(CoreTrainer, 'create_pl_trainer', create_pl_trainer) + monkeypatch.setenv('KUMO_ACCELERATOR', 'mps') + previous = get_scratch_dir() + kumo.configure(scratch_dir=tmp_path / 'scratch') + try: + data = tmp_path / 'data' + data.mkdir() + pd.DataFrame({ + 'id': range(100), + 'label': [bool(i % 2) if i < 80 else None for i in range(100)], + 'value': [i / 100 for i in range(100)], + }).to_parquet(data / 'users.parquet') + users = Table.from_source_table( + FileConnector(root_dir=str(data))['users'], primary_key='id') + graph = Graph({'users': users}) + snapshot = graph.snapshot() + query = PredictiveQuery(graph, 'PREDICT users.label FOR EACH users.id') + plan = ModelPlan() + plan.training_job.num_experiments = 1 + plan.training_job.refit_trainval = True + plan.training_job.disable_explain = True + plan.training_job.disable_compilation = True + plan.optimization.max_epochs = 1 + plan.optimization.min_steps_per_epoch = 2 + plan.optimization.max_steps_per_epoch = 2 + plan.model_architecture.channels = [8] + trainer = Trainer(plan, accelerator='cpu') + result = trainer.fit(query.generate_training_table(snapshot), + snapshot=snapshot) + assert len(devices) >= 2 + fit_count = len(devices) + restored = Trainer.load(result.bundle_dir, accelerator='cpu') + prediction = restored.predict( + query.generate_prediction_table(snapshot), snapshot=snapshot, + binary_classification_threshold=0.5) + assert len(prediction.frame) == 20 + assert len(devices) > fit_count + assert set(devices) == {'cpu'} + finally: + kumo.configure(scratch_dir=previous) diff --git a/packages/kumo-engine/test/test_prediction_completion.py b/packages/kumo-engine/test/test_prediction_completion.py index 46e52e093..a1ad9a53f 100644 --- a/packages/kumo-engine/test/test_prediction_completion.py +++ b/packages/kumo-engine/test/test_prediction_completion.py @@ -57,7 +57,7 @@ def prepared_prediction(snapshot, tmp_path, monkeypatch): lambda *a, **kw: None) monkeypatch.setattr('kumo_core.train.trainer.load_encoders', lambda *_: None) - monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda n: + monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda n, accelerator=None: (None, ) * n) # Real process startup/error propagation is covered by test_prediction_workers; # run the same worker functions here so the real writers/manifests compose. diff --git a/packages/kumo-engine/test/test_prediction_workers.py b/packages/kumo-engine/test/test_prediction_workers.py index fea83408b..c297b5b06 100644 --- a/packages/kumo-engine/test/test_prediction_workers.py +++ b/packages/kumo-engine/test/test_prediction_workers.py @@ -79,7 +79,7 @@ def test_bad_class_count_fails_before_prediction_work(count, monkeypatch): @pytest.mark.parametrize('count', [None, 1, 3]) def test_valid_class_count_reaches_scorer(task_type, count, tmp_path, monkeypatch): - trainer = Trainer(ModelPlan()) + trainer = Trainer(ModelPlan(), accelerator='cpu') trainer._training_result = SimpleNamespace(artifact_paths={}) trainer._training_query = 'trained query' monkeypatch.setattr(Trainer, 'scratch_dir', property(lambda self: tmp_path)) @@ -117,13 +117,16 @@ def test_valid_class_count_reaches_scorer(task_type, count, tmp_path, trainer.predict(table, snapshot=snapshot, num_classes_to_return=count) assert score.call_args.kwargs['top_k'] == count + assert score.call_args.kwargs['accelerator'] == 'cpu' @pytest.mark.parametrize('population,requested,expected', [(7, 3, 3), (1, 3, 1), (0, 3, 0)]) @pytest.mark.parametrize('extra_fields', [None, {'PREDICTIONS': []}]) +@pytest.mark.parametrize('accelerator', [None, 'cpu', 'gpu', 'mps', 'auto']) def test_partition_counts_devices_and_runtime_allocation( - tmp_path, monkeypatch, population, requested, expected, extra_fields): + tmp_path, monkeypatch, population, requested, expected, extra_fields, + accelerator): from kumo_core.resolved_task import ResolvedNumericalTask from kumo_core.train.diskgraph_nested_trainer import DiskgraphNestedTrainer loads, calls = [], [] @@ -151,7 +154,7 @@ def load(*args, **kwargs): monkeypatch.setattr(DiskgraphNestedTrainer, 'load', load) monkeypatch.setattr(prediction, '_prediction_worker_devices', - lambda n: tuple(range(n))) + lambda n, accelerator=None: tuple(range(n))) monkeypatch.setattr( prediction, '_run_prediction_workers', lambda n, score, kw: [score(worker_id=i, **kw) for i in range(n)]) @@ -161,7 +164,7 @@ def load(*args, **kwargs): run_id='run', scratch_dir=str(tmp_path), artifacts=frozenset({PredictionArtifactType.PREDICTIONS}), num_workers=requested, extra_fields=extra_fields, - graph_snapshot_id='snapshot', + accelerator=accelerator, graph_snapshot_id='snapshot', source_config_sha256='2' * 64, embedding_models=store) if not expected: with pytest.raises(RuntimeError, match='wrote no parquet'): @@ -173,6 +176,7 @@ def load(*args, **kwargs): assert len(calls) == expected assert [load['devices'] for load in loads] == [[i] for i in range(expected)] assert all(load['embedding_models'] is store for load in loads) + assert all(load.get('accelerator') == accelerator for load in loads) assert [call['partition_id'] for call in calls] == list(range(expected)) assert [call['output_partition_ids'] for call in calls] == [[i] for i in range(expected)] @@ -194,6 +198,23 @@ def test_cuda_device_admission(monkeypatch): assert prediction._prediction_worker_devices(2) == (None, None) +@pytest.mark.parametrize('accelerator', ['cpu', 'mps']) +def test_explicit_accelerator_skips_cuda_admission(monkeypatch, accelerator): + import torch + monkeypatch.setenv('KUMO_ACCELERATOR', 'gpu') + monkeypatch.setattr(torch.cuda, 'device_count', lambda: 1) + assert prediction._prediction_worker_devices(2, accelerator) == (None, None) + + +@pytest.mark.parametrize('accelerator', ['auto', 'gpu']) +def test_explicit_gpu_selection_overrides_cpu_environment( + monkeypatch, accelerator): + import torch + monkeypatch.setenv('KUMO_ACCELERATOR', 'cpu') + monkeypatch.setattr(torch.cuda, 'device_count', lambda: 2) + assert prediction._prediction_worker_devices(2, accelerator) == (0, 1) + + @pytest.mark.parametrize('mismatch', [False, True]) def test_forecast_options_reach_every_scorer_before_output( tmp_path, monkeypatch, mismatch): @@ -243,7 +264,7 @@ def resolve(*args, **kwargs): predict=lambda **kwargs: predicted.append(kwargs), ) monkeypatch.setattr(DiskgraphNestedTrainer, 'load', lambda *a, **k: trainer) - monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda n: + monkeypatch.setattr(prediction, '_prediction_worker_devices', lambda n, accelerator=None: (None, ) * n) monkeypatch.setattr( prediction, '_run_prediction_workers', diff --git a/packages/kumo-engine/test/test_trainer_registry.py b/packages/kumo-engine/test/test_trainer_registry.py index f8a775afc..931200303 100644 --- a/packages/kumo-engine/test/test_trainer_registry.py +++ b/packages/kumo-engine/test/test_trainer_registry.py @@ -299,15 +299,17 @@ def publish_model(self, *args, **kwargs): assert trainer._training_query is None +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) def test_load_reconstructs_trainer_from_published_bundle( - tmp_path: Path) -> None: + tmp_path: Path, accelerator: str) -> None: bundle = _fake_published_bundle(tmp_path) model_plan = ModelPlan() write_training_spec(bundle, predictive_query='PREDICT COUNT(*)', model_plan=model_plan, snapshot_id='snap-123', training_table_generation=_generation_plan()) - loaded = Trainer.load(bundle) + loaded = Trainer.load(bundle, accelerator=accelerator) + assert loaded._accelerator == accelerator assert loaded._snapshot_id == 'snap-123' assert loaded._training_result is not None @@ -320,8 +322,9 @@ def test_load_reconstructs_trainer_from_published_bundle( assert loaded.training_table_generation_plan == _generation_plan() +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) def test_load_from_connector_fetches_then_uses_local_loader( - tmp_path: Path) -> None: + tmp_path: Path, accelerator: str) -> None: bundle = _fake_published_bundle(tmp_path / 'source') write_training_spec(bundle, predictive_query='PREDICT COUNT(*)', model_plan=ModelPlan(), snapshot_id='snap-123', @@ -331,7 +334,9 @@ def test_load_from_connector_fetches_then_uses_local_loader( sink.publish_model(None, str(bundle), location=location) loaded = Trainer.load_from_connector(_FakeConnector(sink), - location=location) + location=location, + accelerator=accelerator) + assert loaded._accelerator == accelerator assert loaded._snapshot_id == 'snap-123' assert loaded._training_result is not None diff --git a/packages/kumo-engine/test/test_training_explanations.py b/packages/kumo-engine/test/test_training_explanations.py index 240c070cf..5e8ed9df1 100644 --- a/packages/kumo-engine/test/test_training_explanations.py +++ b/packages/kumo-engine/test/test_training_explanations.py @@ -21,14 +21,16 @@ (False, True, False), (False, False, True), ]) +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) def test_fit_runs_shared_explanations_after_training(tmp_path, snapshot, monkeypatch, caplog, staged_seed_dir, disabled, explain_fails, - aggregate_fails): + aggregate_fails, + accelerator): plan = ModelPlan() plan.training_job.disable_explain = disabled - trainer = Trainer(plan) + trainer = Trainer(plan, accelerator=accelerator) monkeypatch.setattr(type(trainer), 'scratch_dir', property(lambda _: tmp_path)) table = TrainingTable( @@ -62,6 +64,8 @@ def aggregate_features(*args): 'kumo.engine.trainer.trainer.aggregate_subgraph_features', aggregate) def train(training_input, **kwargs): + assert kwargs['bindings'].runtime.trainer_runtime_kwargs()[ + 'accelerator'] == accelerator events.append('train') return SimpleNamespace( metrics={'test_accuracy': 0.8}, From 848b8f116fabca649431c6db1f76ba376180d3ec Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 13:42:01 -0700 Subject: [PATCH 03/39] Import Faiss without a type-checking guard --- packages/kumo-core/README.md | 6 +++--- .../src/kumo_core/modeling/retrieval/mips.py | 9 +-------- .../kumo-core/tests/modeling/retrieval/test_mips.py | 13 +++++++------ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/kumo-core/README.md b/packages/kumo-core/README.md index 33d4e62db..505090f6e 100644 --- a/packages/kumo-core/README.md +++ b/packages/kumo-core/README.md @@ -59,9 +59,9 @@ Materialization accepts the selected graph in `TrainingWorkflowRequest`, a `Task Use `kumo_core.types` for value types, `kumo_core.training` for workflow contracts, and `kumo_core.train` for the numerical stack. These initializers expose deliberate lazy APIs without loading PyTorch during package discovery. `MIPSFaiss` uses PyTorch's `MIPSBruteForce` for CPU tensors on macOS, avoiding -Faiss CPU search and its OpenMP runtime conflict with PyTorch. Faiss is loaded -only when a Faiss backend is needed. Brute-force search materializes the full -query-by-key score matrix, so large CPU workloads need additional memory. +Faiss CPU search and its OpenMP runtime conflict with PyTorch. +Brute-force search materializes the full query-by-key score matrix, so large +CPU workloads need additional memory. Linux CPU and CUDA tensors retain their Faiss search paths. For MPS embeddings on Apple Silicon, `MIPSFaiss` uses Faiss's Metal backend. diff --git a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py index e16074dad..bfd9b5846 100644 --- a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py +++ b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py @@ -4,15 +4,12 @@ import sys import warnings from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +import faiss import torch from torch import Tensor from torch_geometric.utils import scatter -if TYPE_CHECKING: - import faiss - import logging logger = logging.getLogger(__name__) @@ -145,8 +142,6 @@ def __init__(self, key: Tensor) -> None: self._cpu_index = MIPSBruteForce(self._key) return - # CPU-only macOS processes must not load Faiss's second OpenMP runtime. - import faiss import faiss.contrib.torch_utils if self._key.is_cuda: @@ -176,8 +171,6 @@ def _search( if self._cpu_index is not None: return self._cpu_index.search(query, k, exclude) - import faiss - query_k = k if exclude is not None: # Over-retrieve `k` to compensate for exclusions: diff --git a/packages/kumo-core/tests/modeling/retrieval/test_mips.py b/packages/kumo-core/tests/modeling/retrieval/test_mips.py index b7dc2241e..f9e907283 100644 --- a/packages/kumo-core/tests/modeling/retrieval/test_mips.py +++ b/packages/kumo-core/tests/modeling/retrieval/test_mips.py @@ -239,14 +239,18 @@ def test_mips_metal_all_keys_excluded() -> None: @pytest.mark.parametrize('with_exclude', [False, True]) -def test_macos_cpu_retrieval_does_not_require_faiss( +def test_macos_cpu_retrieval_does_not_use_faiss_search( monkeypatch: pytest.MonkeyPatch, with_exclude: bool, ) -> None: monkeypatch.setattr(sys.modules[MIPSFaiss.__module__], 'sys', SimpleNamespace(platform='darwin')) - # Importing Faiss must fail if the CPU path attempts to load it. - monkeypatch.setitem(sys.modules, 'faiss', None) + import faiss + + def fail_search(*args, **kwargs): + raise AssertionError('macOS CPU retrieval must not use Faiss search') + + monkeypatch.setattr(faiss, 'knn', fail_search) rhs = torch.eye(4, dtype=torch.float64) lhs = torch.tensor([[1., 2., 3., 4.]], dtype=torch.float64, requires_grad=True) @@ -277,7 +281,6 @@ def test_macos_cpu_and_metal_retrieval_in_one_process( import torch from kumo_core.modeling.retrieval.mips import MIPSBruteForce, MIPSFaiss -assert 'faiss' not in sys.modules torch.manual_seed(123) devices = ['cpu'] if torch.backends.mps.is_available(): @@ -295,8 +298,6 @@ def test_macos_cpu_and_metal_retrieval_in_one_process( assert torch.isfinite(lhs.grad).all() layer = torch.nn.Linear(32, 16) layer(torch.randn(512, 32)).sum().backward() - if device == 'cpu' and len(devices) == 1: - assert 'faiss' not in sys.modules print('CPU and Metal retrieval passed') ''' env = { From e06547d6c5252ed17e3bf047b22601e2de672b00 Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 13:58:54 -0700 Subject: [PATCH 04/39] Fix CI typing, imports, and connector test isolation --- kumo/testing/bigquery.py | 2 ++ .../src/kumo_core/modeling/retrieval/mips.py | 3 +- .../kumo-core/tests/evaluation/test_suite.py | 3 ++ packages/kumo-engine/test/test_accelerator.py | 3 +- .../local/test_databricks_connector.py | 9 ++++- test/testing/test_bigquery_credentials.py | 36 +++++++++++++++++++ 6 files changed, 52 insertions(+), 4 deletions(-) diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index db7ad10be..8e0968d86 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -114,6 +114,8 @@ def withBigQueryCredentials(func: Optional[Callable] = None, def decorator(inner_func: Callable): def _setup(): + if os.getenv('KUMO_RUN_GCP_TESTS') == '0': + pytest.skip('GCP integration tests are disabled') setup_bigquery_credentials(service_account_key) if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and not os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): diff --git a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py index bfd9b5846..bcc56cc54 100644 --- a/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py +++ b/packages/kumo-core/src/kumo_core/modeling/retrieval/mips.py @@ -6,6 +6,7 @@ from abc import ABC, abstractmethod import faiss +import faiss.contrib.torch_utils import torch from torch import Tensor from torch_geometric.utils import scatter @@ -142,8 +143,6 @@ def __init__(self, key: Tensor) -> None: self._cpu_index = MIPSBruteForce(self._key) return - import faiss.contrib.torch_utils - if self._key.is_cuda: self._res = faiss.StandardGpuResources() self._res.setDefaultNullStreamAllDevices() diff --git a/packages/kumo-core/tests/evaluation/test_suite.py b/packages/kumo-core/tests/evaluation/test_suite.py index a9ef5ca4d..963a2d8d9 100644 --- a/packages/kumo-core/tests/evaluation/test_suite.py +++ b/packages/kumo-core/tests/evaluation/test_suite.py @@ -153,7 +153,10 @@ def make_store(device: str) -> MetricStore: metric.update(batch_pred.to(device), batch_target.to(device)) metric = store.get_metric_collection(stage) + assert isinstance(metric, torchmetrics.MetricCollection) for member in metric.values(copy_state=False): + assert isinstance( + member, torchmetrics.classification.BinaryPrecisionRecallCurve) assert member.compute_on_cpu == (device == 'mps') assert member.preds[0].device.type == ( 'cpu' if device == 'mps' else device) diff --git a/packages/kumo-engine/test/test_accelerator.py b/packages/kumo-engine/test/test_accelerator.py index a035ff132..a1dfcdee9 100644 --- a/packages/kumo-engine/test/test_accelerator.py +++ b/packages/kumo-engine/test/test_accelerator.py @@ -29,9 +29,10 @@ def test_accelerator_defaults_to_auto_and_is_instance_local(monkeypatch): def test_cpu_fit_refit_reload_and_predict_override_environment( tmp_path, monkeypatch): - import kumo.engine as kumo from kumo_core.train.trainer import Trainer as CoreTrainer + import kumo.engine as kumo + devices = [] original = CoreTrainer.create_pl_trainer diff --git a/test/connector/local/test_databricks_connector.py b/test/connector/local/test_databricks_connector.py index 1a1913e82..701b69475 100644 --- a/test/connector/local/test_databricks_connector.py +++ b/test/connector/local/test_databricks_connector.py @@ -1652,6 +1652,7 @@ async def test_sql_inflight_reuses_only_matching_queries( release = threading.Event() lock = threading.Lock() queries = [] + closed = [] expected = 1 if same_query else 2 loop = asyncio.get_running_loop() @@ -1667,7 +1668,13 @@ def execute(query): result.fetchall_arrow.return_value = pa.table({'query': [query]}) return result + def close(*args): + # Mock.call_count can lose increments across executor threads. + with lock: + closed.append(True) + context.cursor.return_value.__enter__.return_value.execute.side_effect = execute + context.cursor.return_value.__exit__.side_effect = close requested = ['SELECT 1', 'SELECT 1' if same_query else 'SELECT 2'] method = getattr(connector, entrypoint) tasks = [ @@ -1689,7 +1696,7 @@ def execute(query): assert [str(error) for error in results] == ['query failed'] * 2 else: assert [frame['query'].iloc[0] for frame in results] == requested - assert context.cursor.return_value.__exit__.call_count == expected + assert len(closed) == expected def _metadata_connector(rows=()): diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index a87107373..cb8568b42 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -2,6 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import asyncio import os from unittest import mock @@ -17,6 +18,41 @@ ) +@pytest.mark.parametrize('asynchronous', [False, True]) +@pytest.mark.parametrize('enabled', [False, True]) +def test_bigquery_decorator_respects_gcp_test_switch( + monkeypatch, asynchronous, enabled): + monkeypatch.setenv('KUMO_RUN_GCP_TESTS', '1' if enabled else '0') + monkeypatch.setenv(bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON, + 'existing-credentials.json') + setup = mock.Mock() + called = mock.Mock(return_value='result') + monkeypatch.setattr(testing_bigquery, 'setup_bigquery_credentials', setup) + + async def async_test(): + return called() + + def sync_test(): + return called() + + decorated = testing_bigquery.withBigQueryCredentials( + async_test if asynchronous else sync_test) + + def run(): + return asyncio.run(decorated()) if asynchronous else decorated() + + if enabled: + assert run() == 'result' + setup.assert_called_once() + called.assert_called_once() + else: + with pytest.raises(pytest.skip.Exception, + match='GCP integration tests are disabled'): + run() + setup.assert_not_called() + called.assert_not_called() + + def test_bigquery_rest_identity_uses_workload_exchange(monkeypatch): service_account = ( 'gha-kumo-gcp-tests@example-project.iam.gserviceaccount.com') From ad6d76d441b5964e8159b22a8c825875d01a2e5e Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 14:03:12 -0700 Subject: [PATCH 05/39] Revert unrelated BigQuery test changes --- kumo/testing/bigquery.py | 2 -- test/testing/test_bigquery_credentials.py | 36 ----------------------- 2 files changed, 38 deletions(-) diff --git a/kumo/testing/bigquery.py b/kumo/testing/bigquery.py index 8e0968d86..db7ad10be 100644 --- a/kumo/testing/bigquery.py +++ b/kumo/testing/bigquery.py @@ -114,8 +114,6 @@ def withBigQueryCredentials(func: Optional[Callable] = None, def decorator(inner_func: Callable): def _setup(): - if os.getenv('KUMO_RUN_GCP_TESTS') == '0': - pytest.skip('GCP integration tests are disabled') setup_bigquery_credentials(service_account_key) if (not os.getenv(ENV_GOOGLE_CREDENTIALS_JSON) and not os.path.exists(ENV_GOOGLE_CREDENTIALS_DEFAULT_PATH)): diff --git a/test/testing/test_bigquery_credentials.py b/test/testing/test_bigquery_credentials.py index cb8568b42..a87107373 100644 --- a/test/testing/test_bigquery_credentials.py +++ b/test/testing/test_bigquery_credentials.py @@ -2,7 +2,6 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import asyncio import os from unittest import mock @@ -18,41 +17,6 @@ ) -@pytest.mark.parametrize('asynchronous', [False, True]) -@pytest.mark.parametrize('enabled', [False, True]) -def test_bigquery_decorator_respects_gcp_test_switch( - monkeypatch, asynchronous, enabled): - monkeypatch.setenv('KUMO_RUN_GCP_TESTS', '1' if enabled else '0') - monkeypatch.setenv(bigquery_connector.ENV_GOOGLE_CREDENTIALS_JSON, - 'existing-credentials.json') - setup = mock.Mock() - called = mock.Mock(return_value='result') - monkeypatch.setattr(testing_bigquery, 'setup_bigquery_credentials', setup) - - async def async_test(): - return called() - - def sync_test(): - return called() - - decorated = testing_bigquery.withBigQueryCredentials( - async_test if asynchronous else sync_test) - - def run(): - return asyncio.run(decorated()) if asynchronous else decorated() - - if enabled: - assert run() == 'result' - setup.assert_called_once() - called.assert_called_once() - else: - with pytest.raises(pytest.skip.Exception, - match='GCP integration tests are disabled'): - run() - setup.assert_not_called() - called.assert_not_called() - - def test_bigquery_rest_identity_uses_workload_exchange(monkeypatch): service_account = ( 'gha-kumo-gcp-tests@example-project.iam.gserviceaccount.com') From 98ab785aef2bb96696553feaa28925157fbe2a9d Mon Sep 17 00:00:00 2001 From: Haowei Yu Date: Tue, 20 Oct 2026 14:19:17 -0700 Subject: [PATCH 06/39] Make accelerator explicit through Lightning construction --- packages/kumo-core/src/kumo_core/prediction.py | 12 ++++-------- packages/kumo-core/src/kumo_core/train/trainer.py | 9 +++++---- .../kumo-core/src/kumo_core/training/runtime.py | 7 +++---- .../kumo-core/tests/test_local_training_runtime.py | 2 +- packages/kumo-core/tests/test_prediction.py | 4 ++-- packages/kumo-engine/README.md | 3 +-- packages/kumo-engine/test/test_embedding_models.py | 13 ++++++------- .../kumo-engine/test/test_prediction_workers.py | 6 ++++-- 8 files changed, 26 insertions(+), 30 deletions(-) diff --git a/packages/kumo-core/src/kumo_core/prediction.py b/packages/kumo-core/src/kumo_core/prediction.py index e4dcfaec2..effb6b69e 100644 --- a/packages/kumo-core/src/kumo_core/prediction.py +++ b/packages/kumo-core/src/kumo_core/prediction.py @@ -464,7 +464,7 @@ def _run_prediction_workers( def _prediction_worker_devices( num_workers: int, - accelerator: Optional[str] = None, + accelerator: str = 'auto', ) -> Tuple[Optional[int], ...]: r"""Assign concurrent scorers to distinct visible CUDA devices. @@ -474,8 +474,6 @@ def _prediction_worker_devices( """ if num_workers == 1: return (None, ) - accelerator = (accelerator if accelerator is not None else - os.environ.get('KUMO_ACCELERATOR', 'auto')).lower() if accelerator in {'cpu', 'mps'}: return (None, ) * num_workers @@ -512,7 +510,7 @@ class _PredictionWorkerConfig(NamedTuple): forecast_options: Optional[Dict[str, int]] embedding_models: Optional[EmbeddingModelStore] = None ignore_time_col: bool = False - accelerator: Optional[str] = None + accelerator: str = 'auto' def _score_prediction_shard( @@ -540,9 +538,7 @@ def _score_prediction_shard( name=config.run_id, **(config.forecast_options or {}), ) - load_options = {} - if config.accelerator is not None: - load_options['accelerator'] = config.accelerator + load_options = {'accelerator': config.accelerator} device = config.devices[worker_id] if device is not None: load_options['devices'] = [device] @@ -614,7 +610,7 @@ def score_prediction( forecast_options: Mapping[str, int] | None = None, embedding_models: EmbeddingModelStore | None = None, encoders: Any = None, - accelerator: str | None = None, + accelerator: str = 'auto', ) -> Any: """Score partitions and publish bounded metadata using the hosted protocol. diff --git a/packages/kumo-core/src/kumo_core/train/trainer.py b/packages/kumo-core/src/kumo_core/train/trainer.py index cc129a815..200c413d3 100644 --- a/packages/kumo-core/src/kumo_core/train/trainer.py +++ b/packages/kumo-core/src/kumo_core/train/trainer.py @@ -8,7 +8,6 @@ import logging import logging as std_logging import math -import os import os.path as osp import re import sys @@ -399,6 +398,7 @@ class Trainer(PortableConfigMixin): since the global state doesn't get picked up correctly. raise_fake_trial_error_msg (str, optional): A fake error message to raise for test purpose. + accelerator (str): Lightning accelerator selection. Defaults to auto. enable_compile (bool, optional): If set, the model and head are compiled using torch.compile. (default: :obj:`False`) **kwargs (optional): Additional arguments of @@ -471,6 +471,7 @@ def __init__( artifact_runtime: TrainingArtifactRuntime | None = None, diskgraph_runtime: DiskgraphTrainingRuntime | None = None, embedding_models: EmbeddingModelStore | None = None, + accelerator: str = 'auto', **kwargs, ) -> None: self._embedding_models = embedding_models @@ -648,6 +649,7 @@ def __init__( # `torch.nn.functional` self.loss = loss self.binary_threshold = binary_threshold + self.accelerator = accelerator self.kwargs = kwargs # Allow Trainer instance to be created with deferred initialization via @@ -1450,16 +1452,15 @@ def create_pl_trainer(self): plugins = [plugins] plugins.append(KumoCheckpointIO()) - # Constructor kwargs override the placement defaults, e.g. a - # batch-prediction worker pinned with ``devices=[i]``: + # Prediction workers can pin a device with ``devices=[i]``. placement = { - 'accelerator': os.environ.get('KUMO_ACCELERATOR', 'auto'), 'strategy': 'auto', 'devices': 1, **self.kwargs, } self.pl_trainer = pl.Trainer( + accelerator=self.accelerator, max_epochs=max_epochs, logger=pl_logger, callbacks=callbacks, diff --git a/packages/kumo-core/src/kumo_core/training/runtime.py b/packages/kumo-core/src/kumo_core/training/runtime.py index 8b1489b78..ba53e5fe1 100644 --- a/packages/kumo-core/src/kumo_core/training/runtime.py +++ b/packages/kumo-core/src/kumo_core/training/runtime.py @@ -143,7 +143,7 @@ def __init__( resolved_task: Any, run_name: Optional[str] = None, embedding_models: Optional[EmbeddingModelStore] = None, - accelerator: Optional[str] = None, + accelerator: str = 'auto', ) -> None: self.home_dir = str(Path(home_dir)) self.resolved_task = resolved_task @@ -185,8 +185,7 @@ def load_automl_config(self, job_config: Any) -> Any: def trainer_runtime_kwargs(self) -> Dict[str, Any]: return { - **({'accelerator': self.accelerator} - if self.accelerator is not None else {}), + 'accelerator': self.accelerator, 'artifact_runtime': self.artifact_runtime, 'diskgraph_runtime': self.diskgraph_runtime, 'explain_runtime': self.explain_runtime, @@ -206,7 +205,7 @@ def local_training_bindings( training_input: Any, run_name: Optional[str] = None, embedding_models: Optional[EmbeddingModelStore] = None, - accelerator: Optional[str] = None, + accelerator: str = 'auto', ) -> Any: r"""Binds one resolved training input to this process. diff --git a/packages/kumo-core/tests/test_local_training_runtime.py b/packages/kumo-core/tests/test_local_training_runtime.py index 0bae4abde..d2af58b2b 100644 --- a/packages/kumo-core/tests/test_local_training_runtime.py +++ b/packages/kumo-core/tests/test_local_training_runtime.py @@ -94,7 +94,7 @@ def test_default_bindings_supply_shared_runtimes(tmp_path): kwargs = bindings.runtime.trainer_runtime_kwargs() assert kwargs['artifact_runtime'] is bindings.runtime.artifact_runtime assert kwargs['embedding_models'] is store - assert 'accelerator' not in kwargs + assert kwargs['accelerator'] == 'auto' @pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) diff --git a/packages/kumo-core/tests/test_prediction.py b/packages/kumo-core/tests/test_prediction.py index 14eed4cc2..779cc083f 100644 --- a/packages/kumo-core/tests/test_prediction.py +++ b/packages/kumo-core/tests/test_prediction.py @@ -482,7 +482,7 @@ def thin(spec, **kwargs): def _score_shard_on_device(monkeypatch, tmp_path, devices, worker_id, - accelerator=None): + accelerator='auto'): r"""Run one real prediction-worker load into ``create_pl_trainer``.""" from kumo_core.encoders.tensor_frame_encoder import ( HeteroTensorFrameEncoder, @@ -550,7 +550,7 @@ def predict(self, **_): def test_prediction_worker_pins_lightning_to_its_device(monkeypatch, tmp_path): import pytorch_lightning as pl - monkeypatch.delenv("KUMO_ACCELERATOR", raising=False) + monkeypatch.setenv("KUMO_ACCELERATOR", "cpu") monkeypatch.setattr(pl, "Trainer", lambda **kwargs: kwargs) trainers = _score_shard_on_device(monkeypatch, tmp_path, (None, 3), 1) diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index ef6851ca6..76561d4df 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -43,8 +43,7 @@ trainer = Trainer(model_plan, accelerator="cpu") Accepted values are `auto` (default), `cpu`, `gpu`, and `mps`, using Lightning's device selection. The choice applies to training, refits, explanations, and -prediction, and overrides `KUMO_ACCELERATOR` for this instance. An explicitly -requested accelerator must be available on the host. +prediction. An explicitly requested accelerator must be available on the host. Device selection is local to the instance and is not saved in model bundles. Select it again when loading, for example diff --git a/packages/kumo-engine/test/test_embedding_models.py b/packages/kumo-engine/test/test_embedding_models.py index 760e3accb..3a55f5915 100644 --- a/packages/kumo-engine/test/test_embedding_models.py +++ b/packages/kumo-engine/test/test_embedding_models.py @@ -38,7 +38,6 @@ def isolated(tmp_path, monkeypatch): monkeypatch.setattr(embedding_models, '_default_store', None) monkeypatch.setattr(TokenEmbeddingTable, 'instances', weakref.WeakValueDictionary()) - monkeypatch.setenv('KUMO_ACCELERATOR', 'cpu') @pytest.fixture(scope='module') @@ -48,7 +47,6 @@ def fitted(tmp_path_factory): monkeypatch.setattr('kumo.engine._scratch._root', root / 'scratch') monkeypatch.setattr(embedding_models, 'entry_points', lambda group: []) monkeypatch.setattr(embedding_models, '_INSTALLED', None) - monkeypatch.setenv('KUMO_ACCELERATOR', 'cpu') archive = _release_zip(root / 'glove.6B.zip', monkeypatch) graph = Graph([_users_table(root / 'source')]) snapshot = graph.snapshot() # Snapshots do not need the models. @@ -68,7 +66,7 @@ def fitted(tmp_path_factory): r"embedding model glove.6B-50d, which is not " r"installed. Install it with pip install " r'"kumo-engine\[text\]"'): - Trainer(plan).fit(table, snapshot=snapshot) + Trainer(plan, accelerator='cpu').fit(table, snapshot=snapshot) warnings['failed fit'] = warned.take() plan.column_processing.encoder_overrides = { @@ -76,7 +74,7 @@ def fitted(tmp_path_factory): } store = EmbeddingModelStore() store.add('words', GloVeVectors(archive, cache_dir=root / 'emb')) - trainer = Trainer(plan, embedding_models=store) + trainer = Trainer(plan, embedding_models=store, accelerator='cpu') result = trainer.fit(table, snapshot=snapshot) warnings['fit'] = warned.take() logging.getLogger('kumo_core.encoder_selection').removeHandler(warned) @@ -138,7 +136,7 @@ def test_a_missing_model_fails_fit_before_any_work(fitted, monkeypatch): monkeypatch.setattr( 'kumo_core.training.workflow.materialize_generated_training_task', lambda *args, **kwargs: pytest.fail('fit did work')) - unfitted = Trainer(trainer.model_plan) + unfitted = Trainer(trainer.model_plan, accelerator='cpu') with pytest.raises(EmbeddingModelNotFoundError, match=r"^Column 'users.bio' uses the embedding model " r"'words', which is not in the store\. .*The store is " @@ -174,7 +172,8 @@ def test_load_matches_models_by_identity_not_name(fitted): archive, root, snapshot, _, prediction_table, _, result, _ = fitted renamed = EmbeddingModelStore() renamed.add('renamed', GloVeVectors(archive, cache_dir=root / 'emb')) - loaded = Trainer.load(result.bundle_dir, embedding_models=renamed) + loaded = Trainer.load(result.bundle_dir, embedding_models=renamed, + accelerator='cpu') predictions = loaded.predict(prediction_table, snapshot=snapshot, binary_classification_threshold=.5) assert predictions.num_rows == 20 @@ -183,6 +182,6 @@ def test_load_matches_models_by_identity_not_name(fitted): match=r"^This model was trained with glove.6B 50d, as " r"'words' for 'users.bio'.*store.add\(\"words\", " r'GloVeVectors\("glove.6B.zip", dim=50\)\)'): - Trainer.load(result.bundle_dir).predict( + Trainer.load(result.bundle_dir, accelerator='cpu').predict( prediction_table, snapshot=snapshot, binary_classification_threshold=.5) diff --git a/packages/kumo-engine/test/test_prediction_workers.py b/packages/kumo-engine/test/test_prediction_workers.py index c297b5b06..2b9e7b7c7 100644 --- a/packages/kumo-engine/test/test_prediction_workers.py +++ b/packages/kumo-engine/test/test_prediction_workers.py @@ -123,7 +123,7 @@ def test_valid_class_count_reaches_scorer(task_type, count, tmp_path, @pytest.mark.parametrize('population,requested,expected', [(7, 3, 3), (1, 3, 1), (0, 3, 0)]) @pytest.mark.parametrize('extra_fields', [None, {'PREDICTIONS': []}]) -@pytest.mark.parametrize('accelerator', [None, 'cpu', 'gpu', 'mps', 'auto']) +@pytest.mark.parametrize('accelerator', ['cpu', 'gpu', 'mps', 'auto']) def test_partition_counts_devices_and_runtime_allocation( tmp_path, monkeypatch, population, requested, expected, extra_fields, accelerator): @@ -195,7 +195,9 @@ def test_cuda_device_admission(monkeypatch): prediction._prediction_worker_devices(2) assert prediction._prediction_worker_devices(1) == (None, ) monkeypatch.setenv('KUMO_ACCELERATOR', 'cpu') - assert prediction._prediction_worker_devices(2) == (None, None) + with pytest.raises(ValueError, match='only 1 are visible'): + prediction._prediction_worker_devices(2) + assert prediction._prediction_worker_devices(2, 'cpu') == (None, None) @pytest.mark.parametrize('accelerator', ['cpu', 'mps']) From 4a9b861a44df6641cc00b8626521b17ac4639438 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 17:03:15 +0000 Subject: [PATCH 07/39] Add built-in SVG graph visualization --- packages/kumo-engine/README.md | 16 ++ .../kumo-engine/kumo/engine/_visualization.py | 255 ++++++++++++++++++ packages/kumo-engine/kumo/engine/graph.py | 21 ++ .../kumo-engine/test/test_graph_validation.py | 141 +++++++++- 4 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 packages/kumo-engine/kumo/engine/_visualization.py diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 3847ae5bd..ee8e0bbc1 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -123,6 +123,22 @@ publication copies bundles to an empty destination and rejects symlinks and overlap. Reload a published bundle with `Trainer.load_from_connector(connector, path=...)`. +## Visualize table relationships + +Display a graph's declared tables and key relationships in a notebook, or save the same view as an SVG: + +```python +graph.visualize() # inline notebook display +svg = graph.visualize("relationships.svg", show_cols=True) +``` + +The return value is SVG text with notebook display support. +Visualization uses declared metadata without reading source rows or requiring Graphviz or other rendering dependencies. +The default view shows primary and relationship foreign keys. +`show_cols=True` adds columns up to 10 rows per table, or 5 for graphs with more than 10 tables; keys always remain visible, even above that limit. +A footer counts hidden columns. +Self-relations are rejected because graph snapshots do not support them. + ## Local storage and results Snapshots, generated tables, model bundles, and unexported predictions live under diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py new file mode 100644 index 000000000..51bba6d90 --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -0,0 +1,255 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Dependency-free SVG rendering of graph schema metadata.""" + +from collections import deque +from html import escape +from math import ceil, sqrt +from typing import Mapping, Sequence +from unicodedata import combining, east_asian_width + +from kumoapi.graph import Edge, validate_edge_endpoints +from kumoapi.table import TableDefinition + +from kumo.engine.table import Table + +_COLUMN_LIMIT = 10 +_LARGE_GRAPH_COLUMN_LIMIT = 5 + + +class SVG(str): + """SVG text that displays inline in notebooks.""" + def _repr_svg_(self) -> str: + return str(self) + + +def _wrap(text: str, width: int) -> list[str]: + lines, line, size = [], '', 0 + split, split_size = 0, 0 + for char in text: + code = ord(char) + if not (code in (9, 10, 13) or 32 <= code <= 0xD7FF + or 0xE000 <= code <= 0xFFFD or 0x10000 <= code <= 0x10FFFF): + raise ValueError('Table and column names must contain valid XML ' + 'characters to visualize them.') + char_size = 0 if combining(char) else ( + 2 if east_asian_width(char) in ('W', 'F') or char in 'MWmw' else 1) + while line and size + char_size > width: + if split: + lines.append(line[:split]) + line, size = line[split:], size - split_size + else: + lines.append(line) + line, size = '', 0 + split, split_size = 0, 0 + line += char + size += char_size + if char.isspace() or char in '_-': + split, split_size = len(line), size + return lines + [line] + + +def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, + show_cols: bool) -> SVG: + validate_edge_endpoints(tables, edges) + names = sorted(tables) + edges = sorted(edges, key=lambda e: (e.src_table, e.fkey, e.dst_table)) + definitions: dict[str, TableDefinition] = {} + keys = {name: {} for name in names} + neighbors: dict[str, set[str]] = {name: set() for name in names} + for name in names: + definition = tables[name].definition + if definition is None: + raise ValueError( + f'Table {name!r} has no definition; infer metadata ' + 'or supply its definition before visualizing.') + definitions[name] = definition + columns = [col.name for col in definition.cols] + if len(columns) != len(set(columns)): + raise ValueError(f'Table {name!r} has duplicate column names; ' + 'give each column a unique name.') + if definition.pkey is not None: + if definition.pkey not in columns: + raise ValueError(f'Primary key {name}.{definition.pkey} is ' + 'missing from the table columns.') + keys[name][definition.pkey] = 'PK' + for edge in edges: + src, dst = edge.src_table, edge.dst_table + if src == dst: + raise ValueError(f'Self-relation {src}.{edge.fkey} -> {dst} cannot ' + 'be visualized; self-relations are not supported ' + 'by graph snapshots. Remove this edge first.') + if edge.fkey not in {col.name for col in definitions[src].cols}: + raise ValueError(f'Foreign key {src}.{edge.fkey} is missing ' + 'from the table columns.') + if definitions[dst].pkey is None: + raise ValueError(f'Destination table {dst!r} has no primary key; ' + 'declare one before visualizing this edge.') + keys[src][edge.fkey] = ('PK/FK' + if definitions[src].pkey == edge.fkey else 'FK') + neighbors[src].add(dst) + neighbors[dst].add(src) + + # Breadth-first groups keep related tables close, with a bounded grid + # rather than an arbitrarily wide chain or tall star. + order = [] + remaining = set(names) + while remaining: + root = min(remaining, key=lambda n: (-len(neighbors[n]), n)) + queue = deque([root]) + remaining.remove(root) + while queue: + name = queue.popleft() + order.append(name) + for neighbor in sorted(neighbors[name] & remaining): + remaining.remove(neighbor) + queue.append(neighbor) + ncols = max(1, ceil(sqrt(len(names)))) + nrows = max(1, ceil(len(names) / ncols)) + positions = {name: divmod(index, nrows) for index, name in enumerate(order)} + cards = {} + for name in names: + definition = definitions[name] + pkey = definition.pkey + columns = sorted((col.name for col in definition.cols), key=lambda col: + (col != pkey, col not in keys[name], col)) + limit = (_COLUMN_LIMIT + if len(names) <= 10 else _LARGE_GRAPH_COLUMN_LIMIT) + columns = columns[:max(len(keys[name]), limit if show_cols else 0)] + title = _wrap(name, 27) + header = 24 + 22 * len(title) + rows = [] + ports = {} + y = header + 8 + for column in columns: + lines = _wrap(column, 22) + height = max(36, 16 + 20 * len(lines)) + ports[column] = y + height / 2 + rows.append((column, lines, y, height)) + y += height + hidden = len(definition.cols) - len(columns) + footer = (f'+ {hidden} other column' + + ('s' if hidden != 1 else '') if hidden else + ('No columns' if not columns else '')) + height = y + (30 if footer else 8) + cards[name] = (title, header, rows, ports, footer, height) + + card_width = 304 + gap = 90 + 8 * len(edges) + row_heights = [ + max((cards[name][-1] + for name in names if positions[name][1] == row), default=90) + for row in range(nrows) + ] + distant = [ + edge for edge in edges + if abs(positions[edge.src_table][0] - positions[edge.dst_table][0]) > 1 + ] + top = 40 + 12 * len(distant) + xy = { + name: + (40 + col * (card_width + gap), top + sum(row_heights[:row]) + row * 48) + for name, (col, row) in positions.items() + } + width = (ncols * card_width + (ncols - 1) * gap + 80 + (gap if any( + positions[e.src_table][0] == positions[e.dst_table][0] == ncols - 1 + for e in edges) else 0)) + height = top + sum(row_heights) + (nrows - 1) * 48 + 40 + if not names: + width, height = 304, 112 + svg = [ + f'', + 'Table relationships', + 'Lines connect foreign keys (FK) to primary keys (PK).' + '', f'' + ] + if not names: + svg.append('' + 'No tables') + lane = 0 + for index, edge in enumerate(edges): + src, dst = edge.src_table, edge.dst_table + sx, sy = xy[src] + dx, dy = xy[dst] + sy += cards[src][3][edge.fkey] + dy += cards[dst][3][definitions[dst].pkey] + sc, dc = positions[src][0], positions[dst][0] + source_side = 1 if sc <= dc else -1 + destination_side = 1 if sc >= dc else -1 + sx += card_width if source_side == 1 else 0 + dx += card_width if destination_side == 1 else 0 + start, end = (sx, sy), (dx, dy) + # Opposing ports use distinct horizontal levels so unrelated edges + # on aligned rows do not appear to join across a gutter. + sx, sy = sx + source_side * 12, sy + source_side * 6 + dx, dy = dx + destination_side * 12, dy + destination_side * 6 + if sc == dc: + bend = sx + 24 + index * 8 + path = f'H {bend} V {dy} H {dx}' + else: + direction = 1 if sc < dc else -1 + if abs(sc - dc) == 1: + bend = (sx + dx) / 2 + direction * (index * 8 - len(edges) * 4) + path = (f'H {bend - direction * 8} ' + f'Q {bend} {sy} {bend} {sy + (8 if dy > sy else -8)} ' + f'V {dy - (8 if dy > sy else -8)} ' + f'Q {bend} {dy} {bend + direction * 8} {dy} H {dx}') + if sy == dy: + path = f'H {dx}' + else: + overhead = 24 + lane * 12 + lane += 1 + a = sx + direction * (24 + index * 8) + b = dx - direction * (24 + index * 8) + path = (f'H {a} V {overhead} H {b} ' + f'V {dy} H {dx}') + path = (f'M {start[0]} {start[1]} L {sx} {sy} {path} ' + f'L {end[0]} {end[1]}') + label = escape(f'{src}.{edge.fkey} → ' + f'{dst}.{definitions[dst].pkey}') + svg.append(f'{label}') + for x, y in (start, end): + svg.append(f'') + for name in names: + x, y = xy[name] + title, header, rows, ports, footer, card_height = cards[name] + svg.append(f'' + f'{escape(name)}' + f'') + svg.append(f'' + f'') + for i, line in enumerate(title): + svg.append(f'{escape(line)}' + '') + for column, lines, ry, rh in rows: + role = keys[name].get(column, '') + if role: + svg.append(f'') + for i, line in enumerate(lines): + svg.append(f'{escape(line)}' + '') + if role: + svg.append(f'{role}') + if footer: + svg.append(f'{footer}') + svg.append('') + return SVG(''.join(svg) + '') diff --git a/packages/kumo-engine/kumo/engine/graph.py b/packages/kumo-engine/kumo/engine/graph.py index 53ebcd199..f5c68f559 100644 --- a/packages/kumo-engine/kumo/engine/graph.py +++ b/packages/kumo-engine/kumo/engine/graph.py @@ -76,6 +76,7 @@ snapshot_scratch_dir, ) from kumo.engine._storage import network_storage +from kumo.engine._visualization import SVG from kumo.engine.table import Table _warned_scratch: set[Path] = set() @@ -346,6 +347,26 @@ def validate(self) -> None: """ self.native_graph + def visualize(self, path: str | Path | None = None, *, + show_cols: bool = False) -> SVG: + """Display table relationships inline in notebooks and return SVG text. + + Args: + path: Optional local file to write as UTF-8 SVG, overwriting it. + show_cols: Add columns up to 10 rows per table (5 above 10 tables). + Primary and foreign keys are always included, even above this + limit. False shows only keys. + + Uses declared schemas without reading source data. Self-relations + and missing schema/key definitions raise ValueError. + """ + from kumo.engine._visualization import render + + svg = render(self._named_tables, self.edges, show_cols=show_cols) + if path is not None: + Path(path).write_text(svg, encoding='utf-8') + return svg + def snapshot(self) -> GraphSnapshot: r"""Materialize this graph: read each table's connector, run the result through `kumo-core` to build column stats and the on-disk diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 959cef60c..80fbc9a8b 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -2,9 +2,11 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import re from dataclasses import replace from pathlib import Path -from unittest.mock import patch +from unittest.mock import PropertyMock, patch +from xml.etree import ElementTree import pytest from kumoapi.graph import Edge @@ -162,3 +164,140 @@ def test_omitted_self_relation_preserves_column_type(tables, original_stype): if col['name'] == 'user_id')['stype'] == original_stype assert next(col for col in source.definition.cols if col.name == 'user_id').stype == original_stype + + +def test_visualize_exports_escaped_svg_without_source_access(tmp_path): + users, events = _s3_tables() + alias = ' & customers' + key = 'customer<&"账户' + events = replace( + events, definition=replace( + events.definition, cols=[ + events.definition.cols[0], + Column(key, Stype.ID, Dtype.int64) + ])) + graph = Graph({ + 'purchases': events, + alias: users + }, [Edge('purchases', key, alias)]) + destination = tmp_path / 'relationships.svg' + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + side_effect=AssertionError('source accessed')), \ + patch.object(Table, 'infer_metadata', + side_effect=AssertionError('inference started')), \ + patch.object(Graph, 'snapshot', + side_effect=AssertionError('snapshot started')): + svg = graph.visualize(destination) + assert isinstance(svg, str) + assert svg._repr_svg_() == svg + assert destination.read_text(encoding='utf-8') == svg + root = ElementTree.fromstring(svg) + assert root.tag == '{http://www.w3.org/2000/svg}svg' + text = ''.join(root.itertext()) + assert alias in text and key in text + assert not root.findall('.//{http://www.w3.org/2000/svg}script') + assert all(not name.lower().startswith('on') for element in root.iter() + for name in element.attrib) + assert 's3://schema-only-fixture' not in svg + + +@pytest.mark.parametrize('count, limit', [(10, 10), (11, 5)]) +@pytest.mark.parametrize('show_cols', [False, True]) +def test_visualize_bounds_column_detail(count, limit, show_cols): + users, _ = _s3_tables() + extras = [ + Column(f'feature_{i:02}', Stype.numerical, Dtype.int64) + for i in range(12) + ] + table = replace( + users, definition=replace(users.definition, + cols=[*users.definition.cols, *extras])) + graph = Graph({f'table_{i:02}': table for i in range(count)}) + root = ElementTree.fromstring(graph.visualize(show_cols=show_cols)) + text = ' '.join( + element.text or '' + for element in root.iter('{http://www.w3.org/2000/svg}text')) + visible = limit - 1 if show_cols else 0 + for i in range(12): + assert (f'feature_{i:02}' in text) == (i < visible) + assert re.search(rf'\+\s*{12 - visible} (other )?columns', text) + + +def test_visualize_retains_keys_beyond_column_limit(): + users, events = _s3_tables() + keys = [Column(f'foreign_{i:02}', Stype.ID, Dtype.int64) for i in range(12)] + events = replace( + events, definition=replace(events.definition, + cols=[events.definition.cols[0], *keys])) + graph = Graph([users, events], + [Edge('events', column.name, 'users') for column in keys]) + text = ''.join(ElementTree.fromstring(graph.visualize()).itertext()) + assert all(column.name in text for column in keys) + assert 'event_id' in text and 'id' in text + + +def test_visualize_layout_is_deterministic_with_cycles_and_isolates(): + _, table = _s3_tables() + tables = {name: table for name in ['a', 'b', 'c', 'isolated']} + edges = [ + Edge('a', 'user_id', 'b'), + Edge('b', 'user_id', 'c'), + Edge('c', 'user_id', 'a'), + Edge('a', 'event_id', 'b') + ] + svg = Graph(tables, edges).visualize() + assert svg == Graph(dict(reversed(list(tables.items()))), + list(reversed(edges))).visualize() + root = ElementTree.fromstring(svg) + text = ''.join(root.itertext()) + assert all(name in text for name in tables) + assert all(float(value) > 0 for value in root.attrib['viewBox'].split()[2:]) + titles = { + element.text + for element in root.iter('{http://www.w3.org/2000/svg}title') + } + assert all( + f'{edge.src_table}.{edge.fkey} → {edge.dst_table}.event_id' in titles + for edge in edges) + + +@pytest.mark.parametrize('case, message', [ + ('unresolved', 'definition|infer_metadata'), + ('endpoint', 'missing'), + ('no_primary_key', 'primary key'), + ('missing_primary_key', 'missing|primary key'), + ('missing_foreign_key', 'missing|foreign key'), + ('self_relation', '[Ss]elf'), + ('duplicate_columns', 'duplicate column'), + ('xml_control', 'valid XML'), +]) +def test_visualize_rejects_invalid_metadata(case, message): + users, events = _s3_tables() + edge = Edge('events', 'user_id', 'users') + if case == 'unresolved': + users = replace(users, definition=None) + elif case in ('no_primary_key', 'missing_primary_key'): + users = replace( + users, definition=replace( + users.definition, + pkey=None if case == 'no_primary_key' else 'gone')) + elif case == 'endpoint': + edge = Edge('missing', 'user_id', 'users') + elif case == 'missing_foreign_key': + edge = Edge('events', 'gone', 'users') + elif case == 'self_relation': + edge = Edge('events', 'user_id', 'events') + elif case == 'duplicate_columns': + users = replace( + users, definition=replace(users.definition, + cols=users.definition.cols * 2)) + elif case == 'xml_control': + users = replace(users, table_name='users\x00') + edge = Edge('events', 'user_id', users.table_name) + with pytest.raises(ValueError, match=message): + Graph([users, events], [edge]).visualize() + + +def test_visualize_empty_graph(): + svg = Graph([]).visualize() + assert 'No tables' in ''.join(ElementTree.fromstring(svg).itertext()) From 02147a84e9ca439cba39fa2c4f2074f5c8aea3aa Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 17:24:10 +0000 Subject: [PATCH 08/39] Keep uppercase and Unicode graph labels within cards --- .../kumo-engine/kumo/engine/_visualization.py | 4 ++-- .../kumo-engine/test/test_graph_validation.py | 16 +++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index 51bba6d90..d8938976e 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -7,7 +7,7 @@ from html import escape from math import ceil, sqrt from typing import Mapping, Sequence -from unicodedata import combining, east_asian_width +from unicodedata import combining from kumoapi.graph import Edge, validate_edge_endpoints from kumoapi.table import TableDefinition @@ -34,7 +34,7 @@ def _wrap(text: str, width: int) -> list[str]: raise ValueError('Table and column names must contain valid XML ' 'characters to visualize them.') char_size = 0 if combining(char) else ( - 2 if east_asian_width(char) in ('W', 'F') or char in 'MWmw' else 1) + 2 if code > 127 or char.isupper() or char in 'mw@%&#' else 1) while line and size + char_size > width: if split: lines.append(line[:split]) diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 80fbc9a8b..c3af75966 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -166,10 +166,14 @@ def test_omitted_self_relation_preserves_column_type(tables, original_stype): if col.name == 'user_id').stype == original_stype -def test_visualize_exports_escaped_svg_without_source_access(tmp_path): +@pytest.mark.parametrize('alias, key', [ + (' & customers', 'customer<&"账户'), + ('PURCHASES_AND_TRANSACTIONS', 'O' * 22), + ('Ж' * 27, 'Ш' * 22), +]) +def test_visualize_exports_escaped_svg_without_source_access( + tmp_path, alias, key): users, events = _s3_tables() - alias = ' & customers' - key = 'customer<&"账户' events = replace( events, definition=replace( events.definition, cols=[ @@ -195,6 +199,12 @@ def test_visualize_exports_escaped_svg_without_source_access(tmp_path): assert root.tag == '{http://www.w3.org/2000/svg}svg' text = ''.join(root.itertext()) assert alias in text and key in text + if alias.isupper(): + visible = [ + element.text + for element in root.iter('{http://www.w3.org/2000/svg}text') + ] + assert alias not in visible and key not in visible assert not root.findall('.//{http://www.w3.org/2000/svg}script') assert all(not name.lower().startswith('on') for element in root.iter() for name in element.attrib) From 7e10946ba031afcbcf5b02e59033719f6cf95e02 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 17:38:56 +0000 Subject: [PATCH 09/39] Match graph connections to the UI curved flow --- .../kumo-engine/kumo/engine/_visualization.py | 188 ++++++++++-------- .../kumo-engine/test/test_graph_validation.py | 4 + 2 files changed, 109 insertions(+), 83 deletions(-) diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index d8938976e..dfb8db7c8 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -3,9 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """Dependency-free SVG rendering of graph schema metadata.""" -from collections import deque from html import escape -from math import ceil, sqrt from typing import Mapping, Sequence from unicodedata import combining @@ -91,23 +89,6 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, neighbors[src].add(dst) neighbors[dst].add(src) - # Breadth-first groups keep related tables close, with a bounded grid - # rather than an arbitrarily wide chain or tall star. - order = [] - remaining = set(names) - while remaining: - root = min(remaining, key=lambda n: (-len(neighbors[n]), n)) - queue = deque([root]) - remaining.remove(root) - while queue: - name = queue.popleft() - order.append(name) - for neighbor in sorted(neighbors[name] & remaining): - remaining.remove(neighbor) - queue.append(neighbor) - ncols = max(1, ceil(sqrt(len(names)))) - nrows = max(1, ceil(len(names) / ncols)) - positions = {name: divmod(index, nrows) for index, name in enumerate(order)} cards = {} for name in names: definition = definitions[name] @@ -135,27 +116,75 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, height = y + (30 if footer else 8) cards[name] = (title, header, rows, ports, footer, height) - card_width = 304 - gap = 90 + 8 * len(edges) - row_heights = [ - max((cards[name][-1] - for name in names if positions[name][1] == row), default=90) - for row in range(nrows) - ] - distant = [ - edge for edge in edges - if abs(positions[edge.src_table][0] - positions[edge.dst_table][0]) > 1 - ] - top = 40 + 12 * len(distant) - xy = { - name: - (40 + col * (card_width + gap), top + sum(row_heights[:row]) + row * 48) - for name, (col, row) in positions.items() + # Rank primary-key tables before their dependents, as in the UI's LR flow. + parents = {name: set() for name in names} + for edge in edges: + parents[edge.src_table].add(edge.dst_table) + ranks: dict[str, int] = {} + pending = set(names) + while pending: + ready = [name for name in pending if not parents[name] & pending] + name = min(ready or pending, key=lambda n: (-len(neighbors[n]), n)) + ranks[name] = max( + (ranks[parent] + 1 for parent in parents[name] if parent in ranks), + default=0) + pending.remove(name) + count = max(ranks.values(), default=0) + 1 + layers = [[] for _ in range(count)] + heights = [cards[name][-1] for name in names] + for node, name in enumerate(names): + layers[ranks[name]].append(node) + node_ids = {name: node for node, name in enumerate(names)} + routes = [] + adjacent: dict[int, list[int]] = {node: [] for node in range(len(names))} + for edge in edges: + source, target = ranks[edge.dst_table], ranks[edge.src_table] + route = [node_ids[edge.dst_table]] + # Empty slots carry long edges through a rank without crossing a card. + for rank in range(source + 1, target): + node = len(heights) + heights.append(16) + layers[rank].append(node) + adjacent[node] = [] + route.append(node) + route.append(node_ids[edge.src_table]) + routes.append(route) + if source < target: + for a, b in zip(route, route[1:]): + adjacent[a].append(b) + adjacent[b].append(a) + centers = { + node: index + for layer in layers + for index, node in enumerate(layer) } - width = (ncols * card_width + (ncols - 1) * gap + 80 + (gap if any( - positions[e.src_table][0] == positions[e.dst_table][0] == ncols - 1 - for e in edges) else 0)) - height = top + sum(row_heights) + (nrows - 1) * 48 + 40 + for sweep in (list(range(count)), list(reversed(range(count)))) * 2: + for rank in sweep: + layers[rank].sort( + key=lambda node: (sum(centers[n] for n in adjacent[node]) / len( + adjacent[node]) if adjacent[node] else centers[node], node)) + offset = 0 + for node in layers[rank]: + centers[node] = offset + heights[node] / 2 + offset += heights[node] + 50 + layer_heights = [ + sum(heights[node] for node in layer) + max(0, + len(layer) - 1) * 50 + for layer in layers + ] + feedback = sum(ranks[e.dst_table] >= ranks[e.src_table] for e in edges) + card_width, gap = 304, max(120, 60 + 8 * feedback) + margin = max(80, 32 + 6 * feedback) + top = 40 + 24 * feedback + canvas_height = max(layer_heights, default=90) + xy = {} + for rank, layer in enumerate(layers): + y = top + (canvas_height - layer_heights[rank]) / 2 + for node in layer: + xy[node] = (margin + rank * (card_width + gap), y) + y += heights[node] + 50 + width = count * card_width + (count - 1) * gap + 2 * margin + height = top + canvas_height + 40 if not names: width, height = 304, 112 svg = [ @@ -171,54 +200,45 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, svg.append('' 'No tables') lane = 0 + handles = set() for index, edge in enumerate(edges): - src, dst = edge.src_table, edge.dst_table - sx, sy = xy[src] - dx, dy = xy[dst] - sy += cards[src][3][edge.fkey] - dy += cards[dst][3][definitions[dst].pkey] - sc, dc = positions[src][0], positions[dst][0] - source_side = 1 if sc <= dc else -1 - destination_side = 1 if sc >= dc else -1 - sx += card_width if source_side == 1 else 0 - dx += card_width if destination_side == 1 else 0 - start, end = (sx, sy), (dx, dy) - # Opposing ports use distinct horizontal levels so unrelated edges - # on aligned rows do not appear to join across a gutter. - sx, sy = sx + source_side * 12, sy + source_side * 6 - dx, dy = dx + destination_side * 12, dy + destination_side * 6 - if sc == dc: - bend = sx + 24 + index * 8 - path = f'H {bend} V {dy} H {dx}' + route = routes[index] + sx, sy = xy[route[0]] + dx, dy = xy[route[-1]] + sx += card_width + sy += cards[edge.dst_table][3][definitions[edge.dst_table].pkey] + dy += cards[edge.src_table][3][edge.fkey] + path = f'M {sx} {sy}' + if ranks[edge.dst_table] < ranks[edge.src_table]: + x, y = sx, sy + for node in route[1:]: + nx, ny = (dx, dy) if node == route[-1] else (xy[node][0], + xy[node][1] + + heights[node] / 2) + # UI getBezierPath: horizontal handles, half-gap controls. + half = (nx - x) / 2 + path += f' C {x + half} {y} {nx - half} {ny} {nx} {ny}' + x, y = nx, ny + if node != route[-1]: + path += (f' C {x + card_width / 3} {y} ' + f'{x + 2 * card_width / 3} {y} ' + f'{x + card_width} {y}') + x += card_width else: - direction = 1 if sc < dc else -1 - if abs(sc - dc) == 1: - bend = (sx + dx) / 2 + direction * (index * 8 - len(edges) * 4) - path = (f'H {bend - direction * 8} ' - f'Q {bend} {sy} {bend} {sy + (8 if dy > sy else -8)} ' - f'V {dy - (8 if dy > sy else -8)} ' - f'Q {bend} {dy} {bend + direction * 8} {dy} H {dx}') - if sy == dy: - path = f'H {dx}' - else: - overhead = 24 + lane * 12 - lane += 1 - a = sx + direction * (24 + index * 8) - b = dx - direction * (24 + index * 8) - path = (f'H {a} V {overhead} H {b} ' - f'V {dy} H {dx}') - path = (f'M {start[0]} {start[1]} L {sx} {sy} {path} ' - f'L {end[0]} {end[1]}') - label = escape(f'{src}.{edge.fkey} → ' - f'{dst}.{definitions[dst].pkey}') + overhead = 20 + lane * 24 + bow = 42 + lane * 8 + lane += 1 + path += (f' C {sx + bow} {sy} {sx + bow} {overhead} ' + f'{sx} {overhead} C {sx - gap} {overhead} ' + f'{dx + gap} {overhead} {dx} {overhead} ' + f'C {dx - bow} {overhead} {dx - bow} {dy} {dx} {dy}') + label = escape(f'{edge.src_table}.{edge.fkey} → ' + f'{edge.dst_table}.{definitions[edge.dst_table].pkey}') svg.append(f'{label}') - for x, y in (start, end): - svg.append(f'') + 'stroke="#a1a1a1" stroke-width="1"/>') + handles.update(((sx, sy), (dx, dy))) for name in names: - x, y = xy[name] + x, y = xy[node_ids[name]] title, header, rows, ports, footer, card_height = cards[name] svg.append(f'' f'{escape(name)}' @@ -252,4 +272,6 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, svg.append(f'{footer}') svg.append('') + for x, y in sorted(handles): + svg.append(f'') return SVG(''.join(svg) + '') diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index c3af75966..0254627e9 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -269,6 +269,10 @@ def test_visualize_layout_is_deterministic_with_cycles_and_isolates(): assert all( f'{edge.src_table}.{edge.fkey} → {edge.dst_table}.event_id' in titles for edge in edges) + paths = root.findall('.//{http://www.w3.org/2000/svg}path[@fill="none"]') + assert len(paths) == len(edges) + assert len({path.attrib['d'] for path in paths}) == len(edges) + assert any(re.search('[CQ]', path.attrib['d']) for path in paths) @pytest.mark.parametrize('case, message', [ From f4c8daba5b440d93349b73e6803672b807ccfe2e Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 19:30:30 +0000 Subject: [PATCH 10/39] Add runnable graph visualization notebook --- .../notebooks/graph_visualizer.ipynb | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 packages/kumo-engine/notebooks/graph_visualizer.ipynb diff --git a/packages/kumo-engine/notebooks/graph_visualizer.ipynb b/packages/kumo-engine/notebooks/graph_visualizer.ipynb new file mode 100644 index 000000000..d15b70526 --- /dev/null +++ b/packages/kumo-engine/notebooks/graph_visualizer.ipynb @@ -0,0 +1,183 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "intro", + "metadata": {}, + "source": [ + "# Kumo graph visualizer demo\n", + "\n", + "Select a Python kernel with the combined `feat/graph-visualize` branch installed. Run natively on **Apple Silicon with macOS 14 or newer** using a wheel built from this branch, or in a supported Linux environment. A Mac does not need Docker for this demo.\n", + "\n", + "The notebook installs nothing and uses synthetic CSV files, with no credentials, training, or graph snapshots. It creates a unique `kumo-visualizer-demo-*` directory below the notebook working directory. CSV files and exported SVGs remain there after the kernel stops." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "imports", + "metadata": {}, + "outputs": [], + "source": [ + "import csv\n", + "import sys\n", + "import tempfile\n", + "from pathlib import Path\n", + "\n", + "from kumo.engine import Edge, FileConnector, Graph, Table\n", + "\n", + "if not hasattr(Graph, \"visualize\"):\n", + " raise RuntimeError(\n", + " \"This kernel does not have Graph.visualize(). Select the environment \"\n", + " \"with feat/graph-visualize installed and restart the kernel.\"\n", + " )\n", + "print(f\"Python: {sys.executable}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "synthetic-tables", + "metadata": {}, + "outputs": [], + "source": [ + "demo_directory = Path(tempfile.mkdtemp(prefix=\"kumo-visualizer-demo-\", dir=Path.cwd()))\n", + "data_directory = demo_directory / \"data\"\n", + "data_directory.mkdir()\n", + "\n", + "tables = {\n", + " \"customers\": (\n", + " [\"customer_id\", \"age\", \"loyalty_score\"],\n", + " [(1, 28, 80), (2, 42, 95), (3, 35, 60)],\n", + " ),\n", + " \"orders\": (\n", + " [\"order_id\", \"customer_id\", \"total\"],\n", + " [(101, 1, 45.0), (102, 2, 30.0), (103, 1, 20.0), (104, 3, 15.0)],\n", + " ),\n", + " \"order_items\": (\n", + " [\"item_id\", \"order_id\", \"product_id\", \"quantity\"],\n", + " [(1001, 101, 10, 2), (1002, 101, 11, 1), (1003, 102, 12, 1),\n", + " (1004, 103, 10, 1), (1005, 104, 11, 3)],\n", + " ),\n", + " \"products\": (\n", + " [\"product_id\", \"price\", \"stock\"],\n", + " [(10, 20.0, 100), (11, 5.0, 250), (12, 30.0, 75)],\n", + " ),\n", + "}\n", + "for name, (columns, rows) in tables.items():\n", + " path = data_directory / f\"{name}.csv\"\n", + " with path.open(\"w\", newline=\"\", encoding=\"utf-8\") as stream:\n", + " writer = csv.writer(stream)\n", + " writer.writerow(columns)\n", + " writer.writerows(rows)\n", + " print(path)" + ] + }, + { + "cell_type": "markdown", + "id": "schema-intro", + "metadata": {}, + "source": [ + "## Connect the tables\n", + "\n", + "Declare primary keys and foreign-key relationships. `Table.from_source_table()` reads the small local CSV schemas; visualization then uses this resolved metadata without materializing a graph." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "build-graph", + "metadata": {}, + "outputs": [], + "source": [ + "files = FileConnector(root_dir=str(data_directory))\n", + "customers = Table.from_source_table(files[\"customers\"], primary_key=\"customer_id\")\n", + "orders = Table.from_source_table(files[\"orders\"], primary_key=\"order_id\")\n", + "order_items = Table.from_source_table(files[\"order_items\"], primary_key=\"item_id\")\n", + "products = Table.from_source_table(files[\"products\"], primary_key=\"product_id\")\n", + "\n", + "graph = Graph(\n", + " tables={\n", + " \"Customers\": customers,\n", + " \"Orders\": orders,\n", + " \"OrderItems\": order_items,\n", + " \"Products\": products,\n", + " },\n", + " edges=[\n", + " Edge(\"Orders\", \"customer_id\", \"Customers\"),\n", + " Edge(\"OrderItems\", \"order_id\", \"Orders\"),\n", + " Edge(\"OrderItems\", \"product_id\", \"Products\"),\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "keys-intro", + "metadata": {}, + "source": [ + "## Show relationships\n", + "\n", + "The default view displays primary and foreign keys. The next cell adds non-key columns, bounded to 10 rows per table for graphs of at most 10 tables and 5 rows for larger graphs. Key rows are always retained." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "keys-view", + "metadata": {}, + "outputs": [], + "source": [ + "graph.visualize()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "detail-view", + "metadata": {}, + "outputs": [], + "source": [ + "graph.visualize(show_cols=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "export-svg", + "metadata": {}, + "outputs": [], + "source": [ + "keys_svg = demo_directory / \"relationships.svg\"\n", + "detail_svg = demo_directory / \"relationships-with-columns.svg\"\n", + "graph.visualize(keys_svg)\n", + "graph.visualize(detail_svg, show_cols=True)\n", + "print(f\"Keys-only SVG: {keys_svg}\")\n", + "print(f\"Detailed SVG: {detail_svg}\")" + ] + }, + { + "cell_type": "markdown", + "id": "your-tables", + "metadata": {}, + "source": [ + "## Try your own graph\n", + "\n", + "Replace the four synthetic `Table` objects with your resolved tables and update the aliases and `Edge` definitions. If you already have a `Graph`, call `your_graph.visualize()` directly, or `your_graph.visualize(\"my-graph.svg\", show_cols=True)` to save a detailed view. No `snapshot()` call is needed." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From cbee55d07d9c0c91d8a64c458cb3b12215585a96 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 20:37:51 +0000 Subject: [PATCH 11/39] Preserve spaces in graph visualization labels --- packages/kumo-engine/kumo/engine/_visualization.py | 6 ++++-- packages/kumo-engine/test/test_graph_validation.py | 6 +++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index dfb8db7c8..616657ef7 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -250,7 +250,8 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, f'') for i, line in enumerate(title): - svg.append(f'{escape(line)}' '') for column, lines, ry, rh in rows: @@ -260,7 +261,8 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, f'width="{card_width - 20}" height="{rh - 4}" ' 'rx="3" fill="#f7f7f7"/>') for i, line in enumerate(lines): - svg.append(f'{escape(line)}' '') if role: diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 0254627e9..2e04327ef 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -170,6 +170,7 @@ def test_omitted_self_relation_preserves_column_type(tables, original_stype): (' & customers', 'customer<&"账户'), ('PURCHASES_AND_TRANSACTIONS', 'O' * 22), ('Ж' * 27, 'Ш' * 22), + ('sales data', 'order id'), ]) def test_visualize_exports_escaped_svg_without_source_access( tmp_path, alias, key): @@ -197,7 +198,10 @@ def test_visualize_exports_escaped_svg_without_source_access( assert destination.read_text(encoding='utf-8') == svg root = ElementTree.fromstring(svg) assert root.tag == '{http://www.w3.org/2000/svg}svg' - text = ''.join(root.itertext()) + text = ''.join( + element.text or '' + for element in root.iter('{http://www.w3.org/2000/svg}text') + if element.get('{http://www.w3.org/XML/1998/namespace}space') == 'preserve') assert alias in text and key in text if alias.isupper(): visible = [ From d8b80290937dbbb5a291f1db2eb80a17d2f02d23 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 20:55:57 +0000 Subject: [PATCH 12/39] Bound graph notebook output with scrolling and fullscreen --- packages/kumo-engine/README.md | 2 + .../kumo-engine/kumo/engine/_visualization.py | 39 +++++++++++++++++++ packages/kumo-engine/kumo/engine/graph.py | 3 +- .../kumo-engine/test/test_graph_validation.py | 4 ++ 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 32fe51a5a..5fc24ca25 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -150,6 +150,8 @@ svg = graph.visualize("relationships.svg", show_cols=True) ``` The return value is SVG text with notebook display support. +Notebook HTML output keeps labels at their original size in a scrollable view, capped at 640 pixels high, with a Full screen button when the host permits it. +Interactive controls require trusted HTML output; SVG export retains the complete graph at its original dimensions. Visualization uses declared metadata without reading source rows or requiring Graphviz or other rendering dependencies. The default view shows primary and relationship foreign keys. `show_cols=True` adds columns up to 10 rows per table, or 5 for graphs with more than 10 tables; keys always remain visible, even above that limit. diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index 616657ef7..b3fd00c1d 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -21,6 +21,45 @@ class SVG(str): def _repr_svg_(self) -> str: return str(self) + def _repr_html_(self) -> str: + return ( + '
' + '' + '
' + '
' + self + '
' + '
') + def _wrap(text: str, width: int) -> list[str]: lines, line, size = [], '', 0 diff --git a/packages/kumo-engine/kumo/engine/graph.py b/packages/kumo-engine/kumo/engine/graph.py index f5c68f559..241dc91d4 100644 --- a/packages/kumo-engine/kumo/engine/graph.py +++ b/packages/kumo-engine/kumo/engine/graph.py @@ -349,7 +349,7 @@ def validate(self) -> None: def visualize(self, path: str | Path | None = None, *, show_cols: bool = False) -> SVG: - """Display table relationships inline in notebooks and return SVG text. + """Display scrollable table relationships in notebooks and return SVG. Args: path: Optional local file to write as UTF-8 SVG, overwriting it. @@ -359,6 +359,7 @@ def visualize(self, path: str | Path | None = None, *, Uses declared schemas without reading source data. Self-relations and missing schema/key definitions raise ValueError. + Trusted HTML output offers full screen when the notebook host permits it. """ from kumo.engine._visualization import render diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 2e04327ef..085c1ea56 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -198,6 +198,10 @@ def test_visualize_exports_escaped_svg_without_source_access( assert destination.read_text(encoding='utf-8') == svg root = ElementTree.fromstring(svg) assert root.tag == '{http://www.w3.org/2000/svg}svg' + html = ElementTree.fromstring(svg._repr_html_()) + embedded = html.find('.//{http://www.w3.org/2000/svg}svg') + assert ElementTree.tostring(embedded) == ElementTree.tostring(root) + assert html.find('.//div[@role="region"]').get('tabindex') == '0' text = ''.join( element.text or '' for element in root.iter('{http://www.w3.org/2000/svg}text') From 72335e019de4af83141426d9e791fbb1870de307 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Wed, 21 Oct 2026 21:10:06 +0000 Subject: [PATCH 13/39] Show graph visualization in existing notebooks --- .../colab_example_notebook.ipynb | 1 + .../databricks_example_notebook.ipynb | 1 + .../snowflake_example_notebook.ipynb | 2 +- .../notebooks/demo_netflix_gnn.ipynb | 3 + .../notebooks/graph_visualizer.ipynb | 183 ------------------ 5 files changed, 6 insertions(+), 184 deletions(-) delete mode 100644 packages/kumo-engine/notebooks/graph_visualizer.ipynb diff --git a/example_notebooks/colab_example_notebook.ipynb b/example_notebooks/colab_example_notebook.ipynb index 70d5c98c9..1ced3c124 100644 --- a/example_notebooks/colab_example_notebook.ipynb +++ b/example_notebooks/colab_example_notebook.ipynb @@ -466,6 +466,7 @@ " ],\n", ")\n", "graph.validate()\n", + "display(graph.visualize())\n", "print('Graph ID:', graph.id)\n", "print('Tables:', list(graph._named_tables))\n", "print('Edges:', len(graph.edges))\n", diff --git a/example_notebooks/databricks_example_notebook.ipynb b/example_notebooks/databricks_example_notebook.ipynb index 727527969..0ade7e371 100644 --- a/example_notebooks/databricks_example_notebook.ipynb +++ b/example_notebooks/databricks_example_notebook.ipynb @@ -617,6 +617,7 @@ ")\n", "\n", "graph.validate()\n", + "display(graph.visualize())\n", "print(\"graph id:\", graph.id)" ] }, diff --git a/example_notebooks/snowflake_example_notebook.ipynb b/example_notebooks/snowflake_example_notebook.ipynb index 64679ccf7..bb676e88a 100644 --- a/example_notebooks/snowflake_example_notebook.ipynb +++ b/example_notebooks/snowflake_example_notebook.ipynb @@ -187,7 +187,7 @@ "language": "python" }, "outputs": [], - "source": "graph = kumo.Graph(\n tables=[customer, part, orders, lineitem],\n edges=[\n Edge(src_table=\"ORDERS\", fkey=\"O_CUSTKEY\", dst_table=\"CUSTOMER\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_ORDERKEY\", dst_table=\"ORDERS\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_PARTKEY\", dst_table=\"PART\"),\n ],\n)\n\ngraph.validate()\n\nsnapshot = graph.snapshot()\n\nprint(\"Snapshot:\", snapshot.id)" + "source": "from IPython.display import display\n\ngraph = kumo.Graph(\n tables=[customer, part, orders, lineitem],\n edges=[\n Edge(src_table=\"ORDERS\", fkey=\"O_CUSTKEY\", dst_table=\"CUSTOMER\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_ORDERKEY\", dst_table=\"ORDERS\"),\n Edge(src_table=\"LINEITEM\", fkey=\"L_PARTKEY\", dst_table=\"PART\"),\n ],\n)\n\ngraph.validate()\ndisplay(graph.visualize())\n\nsnapshot = graph.snapshot()\n\nprint(\"Snapshot:\", snapshot.id)" }, { "cell_type": "markdown", diff --git a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb index 003b9428f..b0222c0c5 100644 --- a/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb +++ b/packages/kumo-engine/notebooks/demo_netflix_gnn.ipynb @@ -199,6 +199,8 @@ "metadata": {}, "outputs": [], "source": [ + "from IPython.display import display\n", + "\n", "from kumo.engine import Edge\n", "\n", "graph = kumo.Graph(\n", @@ -209,6 +211,7 @@ " ],\n", ")\n", "graph.validate()\n", + "display(graph.visualize())\n", "print('graph valid. snapshot_id:', graph.id)" ] }, diff --git a/packages/kumo-engine/notebooks/graph_visualizer.ipynb b/packages/kumo-engine/notebooks/graph_visualizer.ipynb deleted file mode 100644 index d15b70526..000000000 --- a/packages/kumo-engine/notebooks/graph_visualizer.ipynb +++ /dev/null @@ -1,183 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "intro", - "metadata": {}, - "source": [ - "# Kumo graph visualizer demo\n", - "\n", - "Select a Python kernel with the combined `feat/graph-visualize` branch installed. Run natively on **Apple Silicon with macOS 14 or newer** using a wheel built from this branch, or in a supported Linux environment. A Mac does not need Docker for this demo.\n", - "\n", - "The notebook installs nothing and uses synthetic CSV files, with no credentials, training, or graph snapshots. It creates a unique `kumo-visualizer-demo-*` directory below the notebook working directory. CSV files and exported SVGs remain there after the kernel stops." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "imports", - "metadata": {}, - "outputs": [], - "source": [ - "import csv\n", - "import sys\n", - "import tempfile\n", - "from pathlib import Path\n", - "\n", - "from kumo.engine import Edge, FileConnector, Graph, Table\n", - "\n", - "if not hasattr(Graph, \"visualize\"):\n", - " raise RuntimeError(\n", - " \"This kernel does not have Graph.visualize(). Select the environment \"\n", - " \"with feat/graph-visualize installed and restart the kernel.\"\n", - " )\n", - "print(f\"Python: {sys.executable}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "synthetic-tables", - "metadata": {}, - "outputs": [], - "source": [ - "demo_directory = Path(tempfile.mkdtemp(prefix=\"kumo-visualizer-demo-\", dir=Path.cwd()))\n", - "data_directory = demo_directory / \"data\"\n", - "data_directory.mkdir()\n", - "\n", - "tables = {\n", - " \"customers\": (\n", - " [\"customer_id\", \"age\", \"loyalty_score\"],\n", - " [(1, 28, 80), (2, 42, 95), (3, 35, 60)],\n", - " ),\n", - " \"orders\": (\n", - " [\"order_id\", \"customer_id\", \"total\"],\n", - " [(101, 1, 45.0), (102, 2, 30.0), (103, 1, 20.0), (104, 3, 15.0)],\n", - " ),\n", - " \"order_items\": (\n", - " [\"item_id\", \"order_id\", \"product_id\", \"quantity\"],\n", - " [(1001, 101, 10, 2), (1002, 101, 11, 1), (1003, 102, 12, 1),\n", - " (1004, 103, 10, 1), (1005, 104, 11, 3)],\n", - " ),\n", - " \"products\": (\n", - " [\"product_id\", \"price\", \"stock\"],\n", - " [(10, 20.0, 100), (11, 5.0, 250), (12, 30.0, 75)],\n", - " ),\n", - "}\n", - "for name, (columns, rows) in tables.items():\n", - " path = data_directory / f\"{name}.csv\"\n", - " with path.open(\"w\", newline=\"\", encoding=\"utf-8\") as stream:\n", - " writer = csv.writer(stream)\n", - " writer.writerow(columns)\n", - " writer.writerows(rows)\n", - " print(path)" - ] - }, - { - "cell_type": "markdown", - "id": "schema-intro", - "metadata": {}, - "source": [ - "## Connect the tables\n", - "\n", - "Declare primary keys and foreign-key relationships. `Table.from_source_table()` reads the small local CSV schemas; visualization then uses this resolved metadata without materializing a graph." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "build-graph", - "metadata": {}, - "outputs": [], - "source": [ - "files = FileConnector(root_dir=str(data_directory))\n", - "customers = Table.from_source_table(files[\"customers\"], primary_key=\"customer_id\")\n", - "orders = Table.from_source_table(files[\"orders\"], primary_key=\"order_id\")\n", - "order_items = Table.from_source_table(files[\"order_items\"], primary_key=\"item_id\")\n", - "products = Table.from_source_table(files[\"products\"], primary_key=\"product_id\")\n", - "\n", - "graph = Graph(\n", - " tables={\n", - " \"Customers\": customers,\n", - " \"Orders\": orders,\n", - " \"OrderItems\": order_items,\n", - " \"Products\": products,\n", - " },\n", - " edges=[\n", - " Edge(\"Orders\", \"customer_id\", \"Customers\"),\n", - " Edge(\"OrderItems\", \"order_id\", \"Orders\"),\n", - " Edge(\"OrderItems\", \"product_id\", \"Products\"),\n", - " ],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "keys-intro", - "metadata": {}, - "source": [ - "## Show relationships\n", - "\n", - "The default view displays primary and foreign keys. The next cell adds non-key columns, bounded to 10 rows per table for graphs of at most 10 tables and 5 rows for larger graphs. Key rows are always retained." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "keys-view", - "metadata": {}, - "outputs": [], - "source": [ - "graph.visualize()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "detail-view", - "metadata": {}, - "outputs": [], - "source": [ - "graph.visualize(show_cols=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "export-svg", - "metadata": {}, - "outputs": [], - "source": [ - "keys_svg = demo_directory / \"relationships.svg\"\n", - "detail_svg = demo_directory / \"relationships-with-columns.svg\"\n", - "graph.visualize(keys_svg)\n", - "graph.visualize(detail_svg, show_cols=True)\n", - "print(f\"Keys-only SVG: {keys_svg}\")\n", - "print(f\"Detailed SVG: {detail_svg}\")" - ] - }, - { - "cell_type": "markdown", - "id": "your-tables", - "metadata": {}, - "source": [ - "## Try your own graph\n", - "\n", - "Replace the four synthetic `Table` objects with your resolved tables and update the aliases and `Edge` definitions. If you already have a `Graph`, call `your_graph.visualize()` directly, or `your_graph.visualize(\"my-graph.svg\", show_cols=True)` to save a detailed view. No `snapshot()` call is needed." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 448526a22144864327b533b9ac5677d290fef959 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Mon, 26 Oct 2026 21:49:56 +0000 Subject: [PATCH 14/39] Add stable graph layout controls and notebook previews --- packages/kumo-engine/README.md | 21 +- .../kumo-engine/kumo/engine/_visualization.py | 292 +++++++++---- .../kumo/engine/_visualization_layout.py | 390 +++++++++++++++++ .../kumo/engine/_visualization_viewer.py | 398 ++++++++++++++++++ packages/kumo-engine/kumo/engine/graph.py | 23 +- .../kumo-engine/test/test_graph_validation.py | 272 +++++++++++- .../test/test_visualization_layout.py | 153 +++++++ 7 files changed, 1445 insertions(+), 104 deletions(-) create mode 100644 packages/kumo-engine/kumo/engine/_visualization_layout.py create mode 100644 packages/kumo-engine/kumo/engine/_visualization_viewer.py create mode 100644 packages/kumo-engine/test/test_visualization_layout.py diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 5fc24ca25..7b9e90b09 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -147,15 +147,30 @@ Display a graph's declared tables and key relationships in a notebook, or save t ```python graph.visualize() # inline notebook display svg = graph.visualize("relationships.svg", show_cols=True) +graph.visualize(layout="compact") # balance crowded groups ``` The return value is SVG text with notebook display support. -Notebook HTML output keeps labels at their original size in a scrollable view, capped at 640 pixels high, with a Full screen button when the host permits it. -Interactive controls require trusted HTML output; SVG export retains the complete graph at its original dimensions. -Visualization uses declared metadata without reading source rows or requiring Graphviz or other rendering dependencies. +Choose Original or Compact from the notebook's layout selector. +The default `layout="auto"` chooses an initial arrangement from graph structure, independently of viewport size. +Use `layout="original"` to retain the layered relationship view. +Each arrangement is computed once; switching layouts does not read source data. +Fullscreen fits the selected arrangement into the larger viewport without moving tables or rerouting relationships. +Use the zoom controls or Ctrl/Cmd+wheel (including trackpad pinch) to zoom, and Fit to show the whole graph again. +Ordinary scrolling moves within an overflowing graph and continues through the notebook when the graph cannot scroll further. +Fit changes only the view, and toolbar controls keep their size during graph zoom. +Interactive controls require trusted HTML output and fullscreen requires permission from the notebook host. +SVG export retains the initial selected arrangement at its full dimensions. +By default, visualization uses declared metadata without reading source rows or requiring Graphviz or other rendering dependencies. +Hover or keyboard-focus a header or column for schema details; static SVG and restricted notebook output retain native tooltips. +Optionally use `graph.visualize(sample_rows=3)` to fetch up to 30 preview rows per table once and show up to three non-null examples independently for each visible column. +Samples have no guaranteed ordering, and previews can run warehouse queries and incur costs; the row limit does not bound bytes scanned. +Unavailable previews and columns with no non-null values in the preview are labeled, and long values are abbreviated. +Samples become part of saved notebook HTML; SVG text and exports always contain metadata only. The default view shows primary and relationship foreign keys. `show_cols=True` adds columns up to 10 rows per table, or 5 for graphs with more than 10 tables; keys always remain visible, even above that limit. A footer counts hidden columns. +Visualization accepts at most 30 tables; larger graphs raise an error so no tables or relationships are silently omitted. Self-relations are rejected because graph snapshots do not support them. ## Local storage and results diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index b3fd00c1d..1771351cb 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -3,11 +3,15 @@ # SPDX-License-Identifier: Apache-2.0 """Dependency-free SVG rendering of graph schema metadata.""" +import json from html import escape +from reprlib import Repr from typing import Mapping, Sequence from unicodedata import combining +from uuid import uuid4 from kumoapi.graph import Edge, validate_edge_endpoints +from kumoapi.source_table import CustomSQLSourceTable from kumoapi.table import TableDefinition from kumo.engine.table import Table @@ -17,48 +21,31 @@ class SVG(str): - """SVG text that displays inline in notebooks.""" + """SVG text with optional notebook-only layout controls and previews.""" + + def __new__(cls, value: str, samples: dict[str, str] | None = None, + namespace: str = '', layouts: dict | None = None, + initial_layout: str = 'original'): + obj = super().__new__(cls, value) + obj._samples = samples or {} + obj._namespace = namespace + obj._layouts = layouts or {} + obj._initial_layout = initial_layout + return obj + def _repr_svg_(self) -> str: return str(self) - def _repr_html_(self) -> str: - return ( - '
' - '' - '
' - '
' + self + '
' - '
') + def _repr_html_(self, *, _highlight_table: bool = False) -> str: + from kumo.engine._visualization_viewer import render_html + + svg = str(self) + for metadata, sampled in self._samples.items(): + svg = svg.replace(metadata, sampled) + if self._namespace: + svg = svg.replace(self._namespace, 'kumo-' + uuid4().hex) + return render_html(svg, 'table' if _highlight_table else 'row', + self._layouts, self._initial_layout) def _wrap(text: str, width: int) -> list[str]: @@ -87,8 +74,86 @@ def _wrap(text: str, width: int) -> list[str]: return lines + [line] +def _squircle(width: float, height: float, radius: float = 18) -> str: + # Cubic corners flatten at the sides, unlike circular rounded rectangles. + r = min(radius, width / 2, height / 2) + return (f'M {r} 0 H {width - r} C {width} 0 {width} 0 {width} {r} ' + f'V {height - r} C {width} {height} {width} {height} ' + f'{width - r} {height} H {r} C 0 {height} 0 {height} ' + f'0 {height - r} V {r} C 0 0 0 0 {r} 0 Z') + + +def _tooltip(text: str, row: str = '', table: str = '', + column: str = '', + connected_tables: Sequence[str] = ()) -> str: + connections = escape(json.dumps(list(connected_tables))) + text = escape(text) + attr = text.replace('\n', ' ') + table = escape(table).replace('\n', ' ') + column = escape(column).replace('\n', ' ') + return (f'{text}') + + +def _preview(table: Table, count: int, + columns: Sequence[str]) -> dict[str, str]: + import pandas as pd + from kumo_core.sources.preview import sample_dataframe + + fallback = dict.fromkeys(columns, '\nSample values: unavailable') + try: + assert table.definition is not None + source = table.definition.source_table + sql = source.sql if isinstance(source, CustomSQLSourceTable) else None + response = table.connector.source_backend.get_sample( + table.connector, table.table_name, 30, sql=sql, timeout=10) + frame = sample_dataframe(response.sample_rows) + if frame is None: + return fallback + short = Repr() + short.maxstring, short.maxlist, short.maxtuple = 120, 5, 5 + for column in columns: + if column not in frame: + continue + values = frame[column].iloc[:30].tolist() + if not values: + fallback[column] = '\nSample values: no rows returned' + continue + labels = [] + for value in values: + null = (value is None or value is pd.NA or value is pd.NaT + or isinstance(value, float) and value != value) + if null: + continue + label = short.repr(value) + labels.append(label if len(label) <= 160 else label[:157] + + '...') + if len(labels) == count: + break + fallback[column] = ( + '\nSample values:\n' + '\n'.join(labels) if labels else + '\nSample values: No non-null values in preview') + except Exception: + # Preview failures must not hide the schema or expose source secrets. + pass + return fallback + + def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, - show_cols: bool) -> SVG: + show_cols: bool, sample_rows: int = 0, layout: str = 'auto') -> SVG: + from kumo.engine._visualization_layout import build_layouts, choose_layout + + if not isinstance(layout, str) or layout not in ( + 'auto', 'original', 'compact'): + raise ValueError('layout must be auto, original, or compact.') + if type(sample_rows) is not int or not 0 <= sample_rows <= 3: + raise ValueError('sample_rows must be an integer from 0 to 3.') + if len(tables) > 30: + raise ValueError('Graph visualization supports at most 30 tables; ' + 'select a smaller graph to visualize.') validate_edge_endpoints(tables, edges) names = sorted(tables) edges = sorted(edges, key=lambda e: (e.src_table, e.fkey, e.dst_table)) @@ -138,13 +203,14 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, if len(names) <= 10 else _LARGE_GRAPH_COLUMN_LIMIT) columns = columns[:max(len(keys[name]), limit if show_cols else 0)] title = _wrap(name, 27) - header = 24 + 22 * len(title) + header = 20 + 22 * len(title) rows = [] ports = {} y = header + 8 for column in columns: - lines = _wrap(column, 22) - height = max(36, 16 + 20 * len(lines)) + # Leave room for the widest PK/FK badge and a 12px text gap. + lines = _wrap(column, 25) + height = 12 + 20 * len(lines) ports[column] = y + height / 2 rows.append((column, lines, y, height)) y += height @@ -152,7 +218,7 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, footer = (f'+ {hidden} other column' + ('s' if hidden != 1 else '') if hidden else ('No columns' if not columns else '')) - height = y + (30 if footer else 8) + height = y + (26 if footer else 4) cards[name] = (title, header, rows, ports, footer, height) # Rank primary-key tables before their dependents, as in the UI's LR flow. @@ -226,18 +292,7 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, height = top + canvas_height + 40 if not names: width, height = 304, 112 - svg = [ - f'', - 'Table relationships', - 'Lines connect foreign keys (FK) to primary keys (PK).' - '', f'' - ] - if not names: - svg.append('' - 'No tables') + original_paths = [] lane = 0 handles = set() for index, edge in enumerate(edges): @@ -271,37 +326,118 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, f'{sx} {overhead} C {sx - gap} {overhead} ' f'{dx + gap} {overhead} {dx} {overhead} ' f'C {dx - bow} {overhead} {dx - bow} {dy} {dx} {dy}') + original_paths.append(path) + handles.update(((sx, sy), (dx, dy))) + + layout_cards = [dict(name=name, width=card_width, + height=cards[name][-1], rank=ranks[name], + x=xy[node_ids[name]][0], y=xy[node_ids[name]][1]) + for name in names] + layout_edges = [dict(source=edge.dst_table, target=edge.src_table, + source_port=cards[edge.dst_table][3][ + definitions[edge.dst_table].pkey], + target_port=cards[edge.src_table][3][edge.fkey]) + for edge in edges] + original = dict(width=width, height=height, + positions={name: list(xy[node_ids[name]]) for name in names}, + paths=original_paths, handles=sorted(handles)) + layouts = build_layouts(layout_cards, layout_edges, original) + selected = (choose_layout(layout_cards, layout_edges, layouts) + if layout == 'auto' else layout) + geometry = layouts[selected] + width, height = geometry['width'], geometry['height'] + samples = {} + uid = 'kumo-' + uuid4().hex + svg = [ + f'', + 'Table relationships', + 'Lines connect foreign keys (FK) to primary keys (PK).' + '', f'', + f'', + f'' + ] + if not names: + svg.append('' + 'No tables') + for index, edge in enumerate(edges): + path = geometry['paths'][index] label = escape(f'{edge.src_table}.{edge.fkey} → ' f'{edge.dst_table}.{definitions[edge.dst_table].pkey}') - svg.append(f'{label}') - handles.update(((sx, sy), (dx, dy))) + endpoints = (f'data-src-port="{cards[edge.src_table][3][edge.fkey]}" ' + f'data-dst-port="{cards[edge.dst_table][3][definitions[edge.dst_table].pkey]}" ') + ' '.join( + f'data-{key}="{escape(value).replace(chr(10), " ")}"' + for key, value in [('src-table', + edge.src_table), ( + 'src-column', + edge.fkey), ('dst-table', edge.dst_table), + ('dst-column', + definitions[edge.dst_table].pkey)]) + svg.append( + f'{label}' + f'') for name in names: - x, y = xy[node_ids[name]] + x, y = geometry['positions'][name] title, header, rows, ports, footer, card_height = cards[name] - svg.append(f'' - f'{escape(name)}' - f'') - svg.append(f'' - f'' + f'' + '' + f'') + svg.append(_tooltip(summary, table=name, connected_tables=connected)) + svg.append(f'') for i, line in enumerate(title): svg.append('{escape(line)}' '') - for column, lines, ry, rh in rows: + svg.append('') + for row_index, (column, lines, ry, rh) in enumerate(rows): role = keys[name].get(column, '') - if role: - svg.append(f'') + col = next(c for c in definition.cols if c.name == column) + detail = (f'{name}.{column}\nType: {col.dtype.value}\n' + f'Semantic type: {col.stype.value}\n' + f'Role: {role or "Column"}') + detail += ''.join(f'\nReferences: {e.dst_table}.' + f'{definitions[e.dst_table].pkey}' + for e in outgoing if e.fkey == column) + row = f'{node_ids[name]}-{row_index}' + metadata = _tooltip(detail, row, name, column) + if column in preview: + samples[metadata] = _tooltip(detail + preview[column], row, + name, column) + svg.append(metadata) + fill = '#f7f7f7' if role else 'transparent' + svg.append(f'') for i, line in enumerate(lines): svg.append('{escape(line)}' '') if role: @@ -309,10 +445,12 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, f'y="{ry + rh / 2 + 4}" text-anchor="end" ' f'font-size="10" font-weight="700" ' f'fill="#386000">{role}') + svg.append('') if footer: svg.append(f'{footer}') svg.append('') - for x, y in sorted(handles): + for x, y in geometry['handles']: svg.append(f'') - return SVG(''.join(svg) + '') + return SVG(''.join(svg) + '', samples, uid, layouts=layouts, + initial_layout=selected) diff --git a/packages/kumo-engine/kumo/engine/_visualization_layout.py b/packages/kumo-engine/kumo/engine/_visualization_layout.py new file mode 100644 index 000000000..e7530880c --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_layout.py @@ -0,0 +1,390 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Deterministic graph geometry, independent of rendering and viewport state. + +All coordinates use SVG units. Edges run from a primary-key table's right +port to a foreign-key table's left port. Layouts never read source data. +""" +from __future__ import annotations + +import heapq +import math +from collections import defaultdict +from typing import TypedDict + + +class Card(TypedDict): + name: str + width: float + height: float + rank: int + x: float + y: float + + +class Edge(TypedDict): + source: str + target: str + source_port: float + target_port: float + + +class Geometry(TypedDict): + width: float + height: float + positions: dict[str, list[float]] + paths: list[str] + handles: list[list[float]] + + +Point = tuple[float, float] +Rect = tuple[float, float, float, float] +_GAP = 72.0 +_COLUMN_GAP = 140.0 +_MARGIN = 80.0 + + +def _components(cards: list[Card], edges: list[Edge]) -> list[list[Card]]: + lookup = {card['name']: card for card in cards} + neighbors: dict[str, set[str]] = {name: set() for name in lookup} + for edge in edges: + neighbors[edge['source']].add(edge['target']) + neighbors[edge['target']].add(edge['source']) + pending = set(lookup) + result = [] + while pending: + stack = [min(pending)] + found = set() + while stack: + name = stack.pop() + if name in found: + continue + found.add(name) + stack.extend(sorted(neighbors[name] - found, reverse=True)) + pending -= found + result.append([lookup[name] for name in sorted(found)]) + return sorted(result, key=lambda group: (-len(group), group[0]['name'])) + + +def _overlap(a: Rect, b: Rect, gap: float = 0) -> bool: + return (a[0] < b[2] + gap and a[2] + gap > b[0] + and a[1] < b[3] + gap and a[3] + gap > b[1]) + + +def _bounds(cards: list[Card], positions: dict[str, list[float]]) -> Rect: + return (min(positions[c['name']][0] for c in cards), + min(positions[c['name']][1] for c in cards), + max(positions[c['name']][0] + c['width'] for c in cards), + max(positions[c['name']][1] + c['height'] for c in cards)) + + +def _compact_component(cards: list[Card], edges: list[Edge] + ) -> dict[str, list[float]]: + ranks: dict[int, list[Card]] = defaultdict(list) + for card in cards: + ranks[card['rank']].append(card) + sparse_hub = len(ranks) <= 2 and len(edges) <= 2 * len(cards) + for group in ranks.values(): + group.sort(key=lambda c: (c['y'], c['name'])) + # Dense cyclic ranks are artifacts of breaking cycles, not useful layers. + # A requested compact view may also wrap a long, single-table chain. + if len(edges) > 2 * len(cards) or len(ranks) > max(4, len(cards) // 2): + ordered = sorted(cards, key=lambda c: (c['rank'], c['y'], c['name'])) + ranks = {0: ordered} + tallest = max(c['height'] for c in cards) + total = sum(c['height'] + _GAP for c in cards) + candidates = [] + for column_count in range(1, min(len(cards), 8) + 1): + limit = max(tallest, total / column_count) + columns: list[list[Card]] = [] + for rank in sorted(ranks): + group = ranks[rank] + parts = min(len(group), max(1, math.ceil( + sum(c['height'] + _GAP for c in group) / (limit + _GAP)))) + if sparse_hub: + # Splitting one crowded stack into two is enough for sparse + # hubs; avoid spreading a simple graph across many columns. + parts = min(parts, 2) + remaining = list(group) + for part in range(parts): + needed = parts - part + target = sum(c['height'] + _GAP for c in remaining) / needed + running, best, stop = 0.0, math.inf, 1 + for at, card in enumerate(remaining[:len(remaining) - needed + 1]): + running += card['height'] + _GAP + if abs(running - target) < best: + best, stop = abs(running - target), at + 1 + columns.append(remaining[:stop]) + remaining = remaining[stop:] + heights = [sum(c['height'] for c in col) + _GAP * (len(col) - 1) + for col in columns] + positions: dict[str, list[float]] = {} + x = 0.0 + for column, height in zip(columns, heights): + y = (max(heights) - height) / 2 + for card in column: + positions[card['name']] = [x, y] + y += card['height'] + _GAP + x += max(c['width'] for c in column) + _COLUMN_GAP + width, height = x - _COLUMN_GAP, max(heights) + # A fixed reading canvas keeps geometry independent of the notebook. + # Count and heights determine packing; viewport fit is a viewer task. + score = max(width / 1.45, height) + .06 * (width + height) + span = sum(math.hypot( + positions[e['source']][0] - positions[e['target']][0], + positions[e['source']][1] + e['source_port'] + - positions[e['target']][1] - e['target_port']) for e in edges) + score += .06 * span / max(1, len(edges)) + candidates.append((score, column_count, positions)) + return min(candidates, key=lambda item: item[:2])[2] + + +def _pack_components(cards: list[Card], edges: list[Edge] + ) -> dict[str, list[float]]: + blocks = [] + for group in _components(cards, edges): + names = {c['name'] for c in group} + relations = [e for e in edges if e['source'] in names] + positions = _compact_component(group, relations) + left, top, right, bottom = _bounds(group, positions) + positions = {name: [x - left, y - top] + for name, (x, y) in positions.items()} + blocks.append((right - left, bottom - top, positions)) + # Place separate domains as rectangles. Small isolated tables can fill + # gaps beside an earlier component instead of forcing another full row. + gap = 96.0 + minimum = max(w for w, _, _ in blocks) + total = sum(w + gap for w, _, _ in blocks) - gap + candidates = [] + for steps in range(9): + limit = minimum + (total - minimum) * steps / 8 + placed: list[Rect] = [] + result: dict[str, list[float]] = {} + for width, height, positions in blocks: + xs = sorted({0.0, *(r[2] + gap for r in placed)}) + ys = sorted({0.0, *(r[3] + gap for r in placed)}) + slots = [(y, x) for y in ys for x in xs + if x + width <= limit + 1e-6 + and not any(_overlap((x, y, x + width, y + height), + r, gap) for r in placed)] + y, x = min(slots) + placed.append((x, y, x + width, y + height)) + for name, (cx, cy) in positions.items(): + result[name] = [cx + x + _MARGIN, cy + y + _MARGIN] + width = max(r[2] for r in placed) + height = max(r[3] for r in placed) + score = max(width / 1.45, height) + .06 * (width + height) + candidates.append((score, steps, result)) + return min(candidates, key=lambda item: item[:2])[2] + + +def _line_hits(a: Point, b: Point, rect: Rect) -> bool: + """Whether an open segment enters the interior of a rectangle.""" + if (max(a[0], b[0]) <= rect[0] or min(a[0], b[0]) >= rect[2] + or max(a[1], b[1]) <= rect[1] or min(a[1], b[1]) >= rect[3]): + return False + low, high = 0.0, 1.0 + for axis in (0, 1): + delta = b[axis] - a[axis] + if abs(delta) < 1e-9: + if a[axis] <= rect[axis] or a[axis] >= rect[axis + 2]: + return False + else: + start = (rect[axis] - a[axis]) / delta + end = (rect[axis + 2] - a[axis]) / delta + low, high = max(low, min(start, end)), min(high, max(start, end)) + if low >= high - 1e-9: + return False + return high > 0 and low < 1 + + +def _curve_clear(curve: tuple[Point, ...], rect: Rect, depth: int = 7) -> bool: + box = (min(p[0] for p in curve), min(p[1] for p in curve), + max(p[0] for p in curve), max(p[1] for p in curve)) + if not _overlap(box, rect): + return True + if depth == 0: + return False + a, b, c, d = curve + ab = ((a[0] + b[0]) / 2, (a[1] + b[1]) / 2) + bc = ((b[0] + c[0]) / 2, (b[1] + c[1]) / 2) + cd = ((c[0] + d[0]) / 2, (c[1] + d[1]) / 2) + abc = ((ab[0] + bc[0]) / 2, (ab[1] + bc[1]) / 2) + bcd = ((bc[0] + cd[0]) / 2, (bc[1] + cd[1]) / 2) + mid = ((abc[0] + bcd[0]) / 2, (abc[1] + bcd[1]) / 2) + return (_curve_clear((a, ab, abc, mid), rect, depth - 1) + and _curve_clear((mid, bcd, cd, d), rect, depth - 1)) + + +def _point(point: Point) -> str: + return f'{point[0]:.3f} {point[1]:.3f}' + + +def _smooth(points: list[Point]) -> str: + # Rounding is smaller than obstacle clearance, so curves cannot cut into + # cards when replacing a visibility-graph corner. + path = 'M ' + _point(points[0]) + for a, b, c in zip(points, points[1:], points[2:]): + before, after = math.dist(a, b), math.dist(b, c) + if min(before, after) < 1e-9: + continue + radius = min(12.0, before * .35, after * .35) + p = (b[0] + (a[0] - b[0]) * radius / before, + b[1] + (a[1] - b[1]) * radius / before) + q = (b[0] + (c[0] - b[0]) * radius / after, + b[1] + (c[1] - b[1]) * radius / after) + path += f' L {_point(p)} Q {_point(b)} {_point(q)}' + return path + ' L ' + _point(points[-1]) + + +class _Router: + """A reusable visibility graph for one arrangement and clearance lane.""" + + def __init__(self, rectangles: list[Rect], clearance: float): + self.rectangles = [(x - clearance, y - clearance, + right + clearance, bottom + clearance) + for x, y, right, bottom in rectangles] + self.points = [(x, y) for left, top, right, bottom in self.rectangles + for x, y in ((left, top), (right, top), + (right, bottom), (left, bottom))] + self.links: list[list[tuple[int, float]]] = [ + [] for _ in self.points] + for i, a in enumerate(self.points): + for j in range(i): + b = self.points[j] + if self.clear(a, b): + distance = math.dist(a, b) + self.links[i].append((j, distance)) + self.links[j].append((i, distance)) + self.connectors: dict[Point, list[tuple[int, float]]] = {} + + def clear(self, a: Point, b: Point) -> bool: + return not any(_line_hits(a, b, r) for r in self.rectangles) + + def connections(self, point: Point) -> list[tuple[int, float]]: + if point not in self.connectors: + self.connectors[point] = [ + (i, math.dist(point, p)) for i, p in enumerate(self.points) + if self.clear(point, p)] + return self.connectors[point] + + def route(self, start: Point, end: Point) -> list[Point]: + if self.clear(start, end): + return [start, end] + finish = dict(self.connections(end)) + distance = [math.inf] * len(self.points) + previous: dict[int, int] = {} + queue = [] + for node, cost in self.connections(start): + distance[node] = cost + heapq.heappush(queue, (cost, node)) + best, last = math.inf, None + while queue: + cost, node = heapq.heappop(queue) + if cost != distance[node]: + continue + if cost >= best: + break + if node in finish and cost + finish[node] < best: + best, last = cost + finish[node], node + for neighbor, step in self.links[node]: + candidate = cost + step + if candidate < distance[neighbor]: + distance[neighbor] = candidate + previous[neighbor] = node + heapq.heappush(queue, (candidate, neighbor)) + if last is None: + raise ValueError('Unable to route graph relationship around tables.') + points = [end] + while last is not None: + points.append(self.points[last]) + last = previous.get(last) + points.append(start) + return points[::-1] + + +def _geometry(cards: list[Card], edges: list[Edge]) -> Geometry: + positions = _pack_components(cards, edges) + lookup = {c['name']: c for c in cards} + rectangles = [(positions[c['name']][0], positions[c['name']][1], + positions[c['name']][0] + c['width'], + positions[c['name']][1] + c['height']) for c in cards] + routers: dict[int, _Router] = {} + paths, handles = [], set() + edge_keys = sorted({(e['source'], e['target'], e['source_port'], + e['target_port']) for e in edges}) + lanes = {key: index % 7 for index, key in enumerate(edge_keys)} + for edge in edges: + source, target = edge['source'], edge['target'] + sx, sy = positions[source] + tx, ty = positions[target] + start = (sx + lookup[source]['width'], sy + edge['source_port']) + end = (tx, ty + edge['target_port']) + handles.update((start, end)) + half = (end[0] - start[0]) / 2 + curve = (start, (start[0] + half, start[1]), + (end[0] - half, end[1]), end) + if half >= 24 and all( + _curve_clear(curve, (rect[0] - 10, rect[1] - 10, + rect[2] + 10, rect[3] + 10)) + for c, rect in zip(cards, rectangles) + if c['name'] not in (source, target)): + paths.append('M ' + _point(start) + ' C ' + + ' '.join(_point(p) for p in curve[1:])) + else: + lane = lanes[(source, target, edge['source_port'], + edge['target_port'])] + if lane not in routers: + routers[lane] = _Router(rectangles, 14 + lane * 3.5) + offset = 40 + lane * 3.5 + route = routers[lane].route( + (start[0] + offset, start[1]), + (end[0] - offset, end[1])) + paths.append(_smooth([start, *route, end])) + _, _, right, bottom = _bounds(cards, positions) + return {'width': right + _MARGIN, 'height': bottom + _MARGIN, + 'positions': positions, 'paths': paths, + 'handles': [list(p) for p in sorted(handles)]} + + +def build_layouts(cards: list[Card], edges: list[Edge], + original: Geometry) -> dict[str, Geometry]: + """Build alternatives once; callers own camera and layout selection. + + The original geometry is passed through unchanged. Input collections are + never mutated, and alternate geometry does not depend on viewport size. + """ + if not cards: + return dict.fromkeys(('original', 'compact'), original) + ordered = sorted(cards, key=lambda c: c['name']) + return {'original': original, 'compact': _geometry(ordered, edges)} + + +def choose_layout(cards: list[Card], edges: list[Edge], + layouts: dict[str, Geometry]) -> str: + """Conservative structural defaults, selected once when rendering.""" + if not cards: + return 'original' + original, compact = layouts['original'], layouts['compact'] + components = _components(cards, edges) + ranks: dict[int, int] = defaultdict(int) + for card in cards: + ranks[card['rank']] += 1 + crowded = max(ranks.values()) >= 7 and original['height'] > 1800 + rank_by_name = {c['name']: c['rank'] for c in cards} + feedback = any(rank_by_name[e['source']] >= rank_by_name[e['target']] + for e in edges) + dense = len(edges) > 2 * len(cards) and feedback + # Keep layered many-to-many dimensions in their familiar two-column + # overview; compact remains available as an explicit alternative. + if len(ranks) == 2 and min(ranks.values()) >= 7: + return 'original' + disconnected = len(components) > 1 + old_extent = max(original['width'] / 1.45, original['height']) + new_extent = max(compact['width'] / 1.45, compact['height']) + improvement = 1.0 if disconnected else .85 + if ((crowded or dense or disconnected) + and new_extent < old_extent * improvement): + return 'compact' + return 'original' diff --git a/packages/kumo-engine/kumo/engine/_visualization_viewer.py b/packages/kumo-engine/kumo/engine/_visualization_viewer.py new file mode 100644 index 000000000..662a6454c --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_viewer.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Self-contained notebook viewer; layout geometry is computed in Python.""" + +import json +from html import escape + +_STYLE = r""" +.kumo-graph{position:relative;display:flex;flex-direction:column; +max-width:100%;padding:8px;box-sizing:border-box;background:white; +color:#202020;font:14px Arial,Helvetica,sans-serif} +.kumo-graph [hidden]{display:none!important} +.kumo-graph:fullscreen{width:100%;height:100%;max-height:none} +.kumo-graph-toolbar{display:flex;align-items:center;justify-content:flex-end; +flex-wrap:wrap;gap:6px;flex:none;margin-bottom:8px} +.kumo-graph-layout-label{display:flex;align-items:center;gap:6px;margin-right:auto; +font:13px Arial,Helvetica,sans-serif;color:#555} +.kumo-graph-layout{height:32px;max-width:140px;font:13px Arial,Helvetica,sans-serif; +color:#333;background:#fff;border:1px solid #d1d1d1;border-radius:8px;padding:0 8px} +.kumo-graph-zoom{display:flex;align-items:center;gap:4px;flex:none} +.kumo-graph-toolbar button{box-sizing:border-box;display:inline-flex; +align-items:center;justify-content:center;flex:0 0 32px; +width:32px!important;min-width:32px!important;max-width:32px!important; +height:32px!important;min-height:32px!important;max-height:32px!important; +padding:6px;margin:0;background:#f5f5f5;color:#555; +border:1px solid #d1d1d1;border-radius:10px;corner-shape:squircle; +cursor:pointer;font:18px/18px Arial,Helvetica,sans-serif} +.kumo-graph-toolbar button:hover{background:#e8e8e8} +.kumo-graph-toolbar button:focus-visible,.kumo-graph-layout:focus-visible{ +outline:2px solid #598d13;outline-offset:2px} +.kumo-graph-toolbar button>svg{display:block;flex:none; +width:18px!important;height:18px!important} +.kumo-graph-scale{min-width:44px;text-align:center;color:#666; +font:12px Arial,Helvetica,sans-serif;font-variant-numeric:tabular-nums} +.kumo-graph .kumo-graph-exit{display:none} +.kumo-graph:fullscreen .kumo-graph-enter{display:none} +.kumo-graph:fullscreen .kumo-graph-exit{display:inline-flex} +.kumo-graph-view{position:relative;overflow:auto;min-height:0;flex:none; +max-width:100%; +background:radial-gradient(circle at 1px 1px,#d9ddd5 .7px, +transparent .8px) 0 0/16px 16px} +.kumo-graph:fullscreen .kumo-graph-view{flex:1;height:auto!important} +.kumo-graph-scene{display:flex;align-items:center;justify-content:center; +min-width:100%;min-height:100%} +.kumo-graph-view svg{display:block;flex:none;max-width:none!important; +max-height:none!important} +.kumo-graph-status{font-size:13px;color:#666;padding:6px 0} +.kumo-graph-view svg>rect{fill:transparent} +.kumo-graph-tooltip{position:absolute;top:0;left:0;z-index:10; +box-sizing:border-box; +width:320px;max-width:calc(100% - 16px);border:1px solid #cbd8ba; +border-radius:18px;corner-shape:squircle;background:#fff;color:#202020; +box-shadow:0 5px 18px #24331416,0 1px 3px #2433140a; +font:16px/1.4 Arial,Helvetica,sans-serif;overflow-wrap:anywhere; +overflow:hidden;pointer-events:none;opacity:0;visibility:hidden; +transform:translateY(4px);transition:opacity 140ms ease,transform 140ms ease, +visibility 140ms ease} +.kumo-graph-tooltip[data-open]{opacity:1;visibility:visible; +transform:translateY(0)} +.kumo-tip-head{background:#f4f8ec;border-bottom:1px solid #e1e9d6; +padding:7px 10px} +.kumo-tip-context{color:#6a7a58;font-size:16px;margin-bottom:2px} +.kumo-tip-name{font-size:17px;font-weight:700;color:#386000} +.kumo-tip-body{padding:7px 10px} +.kumo-tip-badges{display:flex;flex-wrap:wrap;gap:5px} +.kumo-tip-samples{display:flex;gap:5px} +.kumo-tip-badge{background:#f2f4ef;border:1px solid #e1e6db; +border-radius:7px;corner-shape:squircle;padding:2px 6px;color:#536247} +.kumo-tip-key{background:#ecf4df;border-color:#d1e2b7;color:#487014} +.kumo-tip-type{color:#536247} +.kumo-tip-ref{padding-top:5px;color:#5b6751} +.kumo-tip-connections{list-style:none;padding:0;margin:0;display:grid;gap:4px} +.kumo-tip-connection{color:#455239;font-weight:500} +.kumo-tip-label{color:#7a8470;margin:5px 0 3px} +.kumo-tip-sample{flex:1;min-width:0;max-width:100%;white-space:nowrap; +overflow:hidden;text-overflow:ellipsis;background:#f5f7f2;border-radius:7px;corner-shape:squircle; +padding:2px 6px;color:#455239; +font:16px/1.4 ui-monospace,SFMono-Regular,Consolas,monospace} +.kumo-graph [data-kumo-tip]{outline:none;cursor:default} +.kumo-graph .kumo-row-light{fill:#edf6df;stroke:#9bc958;stroke-width:1; +opacity:0;pointer-events:none;transition:opacity 140ms ease} +.kumo-graph .kumo-active .kumo-row-light{opacity:1} +.kumo-graph [data-kumo-tip]:focus-visible .kumo-row-light{ +stroke:#598d13;stroke-width:2} +.kumo-graph .kumo-table-veil{fill:#26341a;opacity:0;pointer-events:none; +transition:opacity 140ms ease} +.kumo-graph[data-highlight=table] .kumo-focused>.kumo-table-veil{opacity:.16} +.kumo-graph[data-highlight=table] .kumo-row-light{fill:transparent} +.kumo-graph .kumo-edge{transition:stroke 140ms ease,stroke-width 140ms ease} +.kumo-graph:fullscreen .kumo-edge{vector-effect:non-scaling-stroke} +.kumo-graph .kumo-edge-active{stroke:#6e9947;stroke-width:1.8} +@media(prefers-reduced-motion:reduce){.kumo-graph-tooltip, +.kumo-graph .kumo-row-light,.kumo-graph .kumo-table-veil, +.kumo-graph .kumo-edge{transition:none}} + +.kumo-graph .kumo-edge{vector-effect:non-scaling-stroke} +""" + +_HOVER = r""" +const tip=document.createElement('div'); +tip.className='kumo-graph-tooltip';tip.role='tooltip'; +tip.id='kumo-tip-'+Math.random().toString(36).slice(2);host.append(tip); +let active=null, card=null, timer; +const element=(cls,text)=>{const n=document.createElement('div'); +n.className='kumo-tip-'+cls;n.textContent=text;return n}; +const hide=()=>{clearTimeout(timer);tip.removeAttribute('data-open'); +edges.forEach(edge=>edge.classList.remove('kumo-edge-active')); +if(active){active.removeAttribute('aria-describedby'); +active.classList.remove('kumo-active')} +card?.classList.remove('kumo-focused');active=card=null}; +const content=el=>{ +const row=!!el.dataset.kumoRow, table=el.dataset.kumoTable; +const name=el.dataset.kumoColumn, full=table+(row?'.'+name:''); +const lines=el.dataset.kumoTip.slice(full.length+1).split('\n'); +const head=element('head',''), body=element('body',''); +head.append(element('context',row?table:'Table'),element('name',row?name:table)); +const badges=element('badges',''); +if(row){ +for(const [cls,prefix] of [[' kumo-tip-key','Role: ']]){ +const value=lines.find(x=>x.startsWith(prefix))?.slice(prefix.length); +if(value)if(value!=='Column')badges.append(element('badge'+cls,value))} +if(badges.childElementCount)body.append(badges); +for(const [prefix,label] of [['Type: ','Data type: '], +['Semantic type: ','Semantic type: ']]){ +const value=lines.find(x=>x.startsWith(prefix))?.slice(prefix.length); +if(value)body.append(element('type',label+value))} +for(const ref of lines.filter(x=>x.startsWith('References: '))) +body.append(element('ref','↗ '+ref.slice(12))); +const index=lines.findIndex(x=>x.startsWith('Sample values:')); +if(index>=0){body.append(element('label','Sample values')); +const status=lines[index].slice(14).trim(), samples=element('samples',''); +for(let value of status?[status]:lines.slice(index+1,index+4)){ +value=value.replace(/^'|'$/g,''); +samples.append(element('sample', +value.length>90?value.slice(0,87)+'…':value))}body.append(samples)} +}else{ +badges.append(element('badge',lines[0])); +body.append(badges); +const connections=JSON.parse(el.dataset.kumoConnections); +body.append(element('label',connections.length?'Connected tables': +'No connected tables')); +if(connections.length){const list=document.createElement('ul'); +list.className='kumo-tip-connections'; +for(const name of connections){const item=document.createElement('li'); +item.className='kumo-tip-connection';item.textContent=name;list.append(item)} +body.append(list)}} +tip.replaceChildren(head,body)}; +const place=()=>{ +if(!active)return; +const h=host.getBoundingClientRect(), a=active.getBoundingClientRect(); +const c=card.getBoundingClientRect(), v=view.getBoundingClientRect(); +let left=Math.max(h.left,0), right=Math.min(h.right,innerWidth); +let top=Math.max(h.top,0), bottom=Math.min(h.bottom,innerHeight); +for(let el=host.matches(':fullscreen')?null:host.parentElement; +el;el=el.parentElement){ +if(el===document.body||el===document.documentElement)break; +const style=getComputedStyle(el), r=el.getBoundingClientRect(); +if(/auto|scroll|hidden|clip/.test(style.overflowX)){ +left=Math.max(left,r.left);right=Math.min(right,r.right)} +if(/auto|scroll|hidden|clip/.test(style.overflowY)){ +top=Math.max(top,r.top);bottom=Math.min(bottom,r.bottom)}} +if(a.right>Math.max(left,v.left))if(Math.min(right,v.right)>a.left) +if(a.bottom>Math.max(top,v.top))if(Math.min(bottom,v.bottom)>a.top){ +left+=8;right-=8;top+=8;bottom-=8; +tip.style.maxWidth=Math.max(0,Math.min(320,right-left))+'px'; +tip.style.maxHeight=Math.max(0,bottom-top)+'px'; +const t=tip.getBoundingClientRect(); +let x=c.right+12, side='right'; +if(x+t.width>right){x=c.left-t.width-12;side='left'} +const raw=x; x=Math.max(left,Math.min(x,right-t.width)); +const y=Math.max(top,Math.min(a.top,bottom-t.height)); +tip.dataset.side=side;tip.dataset.clamped=String(x!==raw||y!==a.top); +tip.style.left=x-h.left+'px';tip.style.top=y-h.top+'px'; +return}hide()}; +const show=el=>{clearTimeout(timer);if(active===el)return; +hide();active=el;card=el.parentElement; +el.classList.add('kumo-active');card.classList.add('kumo-focused'); +content(el);el.setAttribute('aria-describedby',tip.id); +if(el.dataset.kumoRow){ +const table=el.dataset.kumoTable, column=el.dataset.kumoColumn; +edges.forEach(edge=>{const d=edge.dataset; +const src=d.srcTable===table?d.srcColumn===column:false; +const dst=d.dstTable===table?d.dstColumn===column:false; +edge.classList.toggle('kumo-edge-active',src||dst)})} +tip.setAttribute('data-open','');place()}; +view.querySelectorAll('title').forEach(el=>el.remove()); +view.querySelectorAll('svg>g[transform]').forEach(g=>{ +const veil=g.querySelector(':scope>path').cloneNode(); +veil.classList.add('kumo-table-veil');veil.removeAttribute('fill'); +veil.removeAttribute('stroke');g.append(veil)}); +view.querySelectorAll('[data-kumo-tip]').forEach(el=>{ +const light=el.querySelector('.kumo-surface').cloneNode(); +light.classList.add('kumo-row-light');light.removeAttribute('fill'); +light.removeAttribute('stroke');el.insertBefore(light,el.querySelector('text')); +el.addEventListener('mouseenter',()=>show(el),{signal}); +el.addEventListener('mouseleave',()=>{timer=setTimeout(()=>{ +if(active!==document.activeElement)hide()},65)},{signal}); +el.addEventListener('focus',()=>show(el),{signal}); +el.addEventListener('blur',hide,{signal})}); +view.addEventListener('scroll',()=>{ +view.style.backgroundPosition=(-view.scrollLeft)+'px '+(-view.scrollTop)+'px';place()},{signal}); + +""" + +_CAMERA = r""" +const selector=host.querySelector('.kumo-graph-layout'); +const label=host.querySelector('.kumo-graph-scale'); +const status=host.querySelector('[role=status]'); +const enter=host.querySelector('.kumo-graph-enter'); +const exit=host.querySelector('.kumo-graph-exit'); +const payload=JSON.parse(host.querySelector('.kumo-graph-layout-data').textContent); +const layouts=payload.layouts; +let selected=payload.initial, scale=1, fitted=true, full=false, inlineCamera=null; +let layoutWidth=Number(svg.getAttribute('width'))||304; +let layoutHeight=Number(svg.getAttribute('height'))||112; +let resizeFrame=0, disposed=false; +const cards=new Map(Array.from(svg.querySelectorAll(':scope>[data-kumo-card]'), +el=>[el.dataset.kumoCard,el])); +const controls=host.querySelectorAll('button,select'); +controls.forEach(el=>el.disabled=false); +selector.value=selected; +if(Object.keys(layouts).length<2)selector.closest('label').hidden=true; + +// Only applyLayout changes graph geometry. Camera actions never call it. +const applyLayout=name=>{ +const layout=layouts[name];if(!layout)return; +hide(); +for(const [name,position] of Object.entries(layout.positions)){ +const card=cards.get(name); +if(card)card.setAttribute('transform',`translate(${position[0]} ${position[1]})`)} +edges.forEach((edge,index)=>edge.setAttribute('d',layout.paths[index])); +svg.querySelectorAll(':scope>circle,:scope>.kumo-graph-handles').forEach(el=>el.remove()); +const handles=document.createElementNS(svg.namespaceURI,'g'); +handles.setAttribute('class','kumo-graph-handles'); +for(const [x,y] of layout.handles){ +const circle=document.createElementNS(svg.namespaceURI,'circle'); +for(const [key,value] of Object.entries({cx:x,cy:y,r:4,fill:'#64748b'})) +circle.setAttribute(key,value);handles.append(circle)} +svg.append(handles);layoutWidth=layout.width;layoutHeight=layout.height; +svg.setAttribute('viewBox',`0 0 ${layoutWidth} ${layoutHeight}`); +svg.setAttribute('width',layoutWidth);svg.setAttribute('height',layoutHeight); +svg.querySelectorAll(':scope>rect').forEach(rect=>{ +rect.setAttribute('width',layoutWidth);rect.setAttribute('height',layoutHeight)}); +selected=name;host.dataset.layout=name; +}; +const inlineHeight=()=>Math.min(720,Math.max(240,layoutHeight+32)); +// Leave one extra pixel per side beyond the scene's 16px padding. +// Integer client metrics can otherwise round a fit past a fractional edge +// and repeatedly add/remove scrollbars as the observer refits the graph. +const fitScale=()=>Math.min(1, +Math.max(1,view.clientWidth-34)/layoutWidth, +Math.max(1,view.clientHeight-34)/layoutHeight); +const resizeScene=()=>{ +svg.style.width=layoutWidth*scale+'px';svg.style.height=layoutHeight*scale+'px'; +// CSS fills the viewport without rounding its fractional content box. +// Flooring the scene at clientWidth/clientHeight can toggle both scrollbars +// forever when a scrollbar changes the available fractional width. +scene.style.width=layoutWidth*scale+32+'px'; +scene.style.height=layoutHeight*scale+32+'px'; +label.textContent=Math.round(scale*100)+'%'; +host.dataset.scale=String(scale); +}; +const fit=()=>{ +hide();fitted=true;scale=fitScale();resizeScene(); +view.scrollLeft=(scene.offsetWidth-view.clientWidth)/2; +view.scrollTop=(scene.offsetHeight-view.clientHeight)/2; +}; +const zoom=(next,x=view.clientWidth/2,y=view.clientHeight/2)=>{ +hide();fitted=false; +const oldX=(scene.offsetWidth-layoutWidth*scale)/2; +const oldY=(scene.offsetHeight-layoutHeight*scale)/2; +const graphX=(view.scrollLeft+x-oldX)/scale; +const graphY=(view.scrollTop+y-oldY)/scale; +scale=Math.max(Math.min(.1,fitScale()),Math.min(4,next));resizeScene(); +view.scrollLeft=(scene.offsetWidth-layoutWidth*scale)/2+graphX*scale-x; +view.scrollTop=(scene.offsetHeight-layoutHeight*scale)/2+graphY*scale-y; +}; +selector.addEventListener('change',()=>{ +applyLayout(selector.value);if(!full)view.style.height=inlineHeight()+'px';fit(); +},{signal}); +host.querySelector('.kumo-graph-fit').addEventListener('click',fit,{signal}); +host.querySelector('.kumo-graph-zoom-in').addEventListener('click',()=>zoom(scale*1.25),{signal}); +host.querySelector('.kumo-graph-zoom-out').addEventListener('click',()=>zoom(scale/1.25),{signal}); +view.addEventListener('wheel',e=>{ +if(!e.ctrlKey&&!e.metaKey)return; +e.preventDefault();const box=view.getBoundingClientRect(); +const delta=e.deltaY*(e.deltaMode===1?16:e.deltaMode===2?view.clientHeight:1); +zoom(scale*Math.exp(-delta*.002),e.clientX-box.left,e.clientY-box.top); +},{passive:false,signal}); +enter.addEventListener('click',()=>{ +status.hidden=true; +// Capture before fullscreen CSS changes viewport dimensions and clamps scrolling. +inlineCamera={selected,scale,fitted,left:view.scrollLeft,top:view.scrollTop}; +const denied=()=>{status.hidden=false;inlineCamera=null}; +try{Promise.resolve(host.requestFullscreen()).catch(denied)} +catch(error){denied()} +},{signal}); +exit.addEventListener('click',()=>{ +if(document.fullscreenElement===host)document.exitFullscreen(); +},{signal}); +host.addEventListener('fullscreenchange',()=>{ +const entering=document.fullscreenElement===host; +if(entering===full)return; +hide();full=entering; +if(full){ +if(!inlineCamera)inlineCamera={selected,scale,fitted,left:view.scrollLeft,top:view.scrollTop}; +fit();exit.focus({preventScroll:true}); +}else{ +view.style.height=inlineHeight()+'px'; +if(inlineCamera&&inlineCamera.selected===selected){ +scale=inlineCamera.scale;fitted=inlineCamera.fitted;resizeScene(); +view.scrollLeft=inlineCamera.left;view.scrollTop=inlineCamera.top; +}else fit(); +inlineCamera=null;enter.focus({preventScroll:true}); +} +},{signal}); +document.addEventListener('keydown',e=>{if(e.key==='Escape')hide()},{signal}); +document.addEventListener('scroll',place,{capture:true,signal}); +window.addEventListener('resize',place,{signal}); +const resizeObserver=new ResizeObserver(()=>{ +if(disposed)return; +cancelAnimationFrame(resizeFrame); +resizeFrame=requestAnimationFrame(()=>{ +if(fitted)fit();else{hide();resizeScene()} +}); +}); +const dispose=()=>{ +if(disposed)return;disposed=true;hide();controller.abort(); +cancelAnimationFrame(resizeFrame);resizeObserver.disconnect();removalObserver.disconnect(); +host.dataset.disposed='true'; +}; +// Notebook outputs can disappear without any later scroll or keyboard event. +const removalObserver=new MutationObserver(records=>{ +if(records.some(record=>record.removedNodes.length)&&!host.isConnected)dispose(); +}); +removalObserver.observe(host.getRootNode(),{childList:true,subtree:true}); +view.style.height=inlineHeight()+'px';host.dataset.layout=selected; +fit();resizeObserver.observe(view); +""" + + +def render_html(svg: str, mode: str, layouts: dict, initial_layout: str) -> str: + """Wrap one SVG with controls and serialized, precomputed geometries.""" + payload = json.dumps({'layouts': layouts, 'initial': initial_layout}, + ensure_ascii=True, separators=(',', ':')) + payload = payload.replace('<', r'\u003c').replace('>', r'\u003e') + payload = payload.replace('&', r'\u0026') + options = ''.join( + f'' + for name, label in [('original', 'Original'), ('compact', 'Compact')] + if name in layouts) + if not options: + options = '' + fit_icon = ('M 3 8 V 3 H 8 M 16 3 H 21 V 8 M 21 16 V 21 H 16 ' + 'M 8 21 H 3 V 16 M 7 7 H 17 V 17 H 7 Z') + enter_icon = ('M 9 3 H 3 V 9 M 15 3 H 21 V 9 M 3 15 V 21 H 9 ' + 'M 21 15 V 21 H 15') + exit_icon = ('M 3 9 H 9 V 3 M 15 3 V 9 H 21 M 9 21 V 15 H 3 ' + 'M 21 15 H 15 V 21') + + def button(cls: str, label: str, icon: str = '', text: str = '') -> str: + graphic = (f'' + if icon else text) + return (f'') + + return ( + f'
' + f'
' + '' + '
' + + button('zoom-out', 'Zoom out', text='−') + + '' + + button('zoom-in', 'Zoom in', text='+') + + button('fit', 'Fit graph', fit_icon) + '
' + + button('enter', 'Enter full screen', enter_icon) + + button('exit', 'Exit full screen', exit_icon) + '
' + '' + '
' + f'
{svg}
' + f'
') diff --git a/packages/kumo-engine/kumo/engine/graph.py b/packages/kumo-engine/kumo/engine/graph.py index 241dc91d4..a866131eb 100644 --- a/packages/kumo-engine/kumo/engine/graph.py +++ b/packages/kumo-engine/kumo/engine/graph.py @@ -348,7 +348,8 @@ def validate(self) -> None: self.native_graph def visualize(self, path: str | Path | None = None, *, - show_cols: bool = False) -> SVG: + show_cols: bool = False, sample_rows: int = 0, + layout: str = 'auto') -> SVG: """Display scrollable table relationships in notebooks and return SVG. Args: @@ -356,14 +357,28 @@ def visualize(self, path: str | Path | None = None, *, show_cols: Add columns up to 10 rows per table (5 above 10 tables). Primary and foreign keys are always included, even above this limit. False shows only keys. - - Uses declared schemas without reading source data. Self-relations + sample_rows: Show 1 to 3 non-null examples per column in HTML + tooltips, selected from up to 30 preview rows per table. + Zero (default) reads no source data. Previews may + run warehouse queries and incur costs; returned row limits + do not bound bytes scanned. Unavailable previews are labeled. + Samples become part of saved notebook HTML, never SVG exports. + layout: Initial arrangement: auto, original, or compact. + Auto chooses from graph structure, independently of viewport + size. Original preserves the layered relationship view; + compact balances crowded groups. + HTML offers a selector to compare arrangements. + + Uses declared schemas. More than 30 tables, self-relations, and missing schema/key definitions raise ValueError. Trusted HTML output offers full screen when the notebook host permits it. + Full screen, Fit, and zoom preserve the selected arrangement. + SVG exports use the initial layout and contain no interactive controls. """ from kumo.engine._visualization import render - svg = render(self._named_tables, self.edges, show_cols=show_cols) + svg = render(self._named_tables, self.edges, show_cols=show_cols, + sample_rows=sample_rows, layout=layout) if path is not None: Path(path).write_text(svg, encoding='utf-8') return svg diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 085c1ea56..7dc076447 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -5,12 +5,17 @@ import re from dataclasses import replace from pathlib import Path -from unittest.mock import PropertyMock, patch +from unittest.mock import Mock, PropertyMock, patch from xml.etree import ElementTree import pytest from kumoapi.graph import Edge -from kumoapi.source_table import FileType, S3RelativeSource, S3SourceTable +from kumoapi.source_table import ( + CustomSQLSourceTable, + FileType, + S3RelativeSource, + S3SourceTable, +) from kumoapi.table import Column, TableDefinition from kumoapi.typing import Dtype, Stype @@ -199,13 +204,19 @@ def test_visualize_exports_escaped_svg_without_source_access( root = ElementTree.fromstring(svg) assert root.tag == '{http://www.w3.org/2000/svg}svg' html = ElementTree.fromstring(svg._repr_html_()) + assert html.get('data-highlight') == 'row' + shaded = ElementTree.fromstring(svg._repr_html_(_highlight_table=True)) + assert shaded.get('data-highlight') == 'table' embedded = html.find('.//{http://www.w3.org/2000/svg}svg') - assert ElementTree.tostring(embedded) == ElementTree.tostring(root) + assert re.sub(rb'kumo-[0-9a-f]{32}', b'kumo-ID', + ElementTree.tostring(embedded)) == re.sub( + rb'kumo-[0-9a-f]{32}', b'kumo-ID', + ElementTree.tostring(root)) assert html.find('.//div[@role="region"]').get('tabindex') == '0' - text = ''.join( - element.text or '' - for element in root.iter('{http://www.w3.org/2000/svg}text') - if element.get('{http://www.w3.org/XML/1998/namespace}space') == 'preserve') + text = ''.join(element.text or '' + for element in root.iter('{http://www.w3.org/2000/svg}text') + if element.get('{http://www.w3.org/XML/1998/namespace}space') + == 'preserve') assert alias in text and key in text if alias.isupper(): visible = [ @@ -216,16 +227,28 @@ def test_visualize_exports_escaped_svg_without_source_access( assert not root.findall('.//{http://www.w3.org/2000/svg}script') assert all(not name.lower().startswith('on') for element in root.iter() for name in element.attrib) + tips = root.findall('.//{http://www.w3.org/2000/svg}g[@data-kumo-tip]') + assert len(tips) == 5 # Two headers, three visible key rows. + assert all(tip.get('tabindex') == '0' for tip in tips) + details = [tip.get('data-kumo-tip') for tip in tips] + assert any(f'References: {alias}.id' in tip for tip in details) + assert any('Role: FK' in tip for tip in details) + assert any('Type: int64\nSemantic type: ID\nRole: FK' in tip + for tip in details) + assert any(f'Connected tables:\n{alias}' in tip for tip in details) + edge = root.find('.//{http://www.w3.org/2000/svg}path[@class="kumo-edge"]') + assert edge.get('data-dst-table') == alias + assert edge.get('data-src-column') == key assert 's3://schema-only-fixture' not in svg -@pytest.mark.parametrize('count, limit', [(10, 10), (11, 5)]) +@pytest.mark.parametrize('count, limit', [(1, 10), (10, 10), (11, 5)]) @pytest.mark.parametrize('show_cols', [False, True]) def test_visualize_bounds_column_detail(count, limit, show_cols): users, _ = _s3_tables() extras = [ - Column(f'feature_{i:02}', Stype.numerical, Dtype.int64) - for i in range(12) + Column(f'feature_{i:03}', Stype.numerical, Dtype.int64) + for i in range(120) ] table = replace( users, definition=replace(users.definition, @@ -236,9 +259,9 @@ def test_visualize_bounds_column_detail(count, limit, show_cols): element.text or '' for element in root.iter('{http://www.w3.org/2000/svg}text')) visible = limit - 1 if show_cols else 0 - for i in range(12): - assert (f'feature_{i:02}' in text) == (i < visible) - assert re.search(rf'\+\s*{12 - visible} (other )?columns', text) + for i in range(120): + assert (f'feature_{i:03}' in text) == (i < visible) + assert re.search(rf'\+\s*{120 - visible} (other )?columns', text) def test_visualize_retains_keys_beyond_column_limit(): @@ -254,7 +277,10 @@ def test_visualize_retains_keys_beyond_column_limit(): assert 'event_id' in text and 'id' in text -def test_visualize_layout_is_deterministic_with_cycles_and_isolates(): +@pytest.mark.parametrize('ambiguous', [False, True]) +@pytest.mark.parametrize('layout', ['auto', 'original', 'compact']) +def test_visualize_layout_is_deterministic_with_cycles_and_isolates( + ambiguous, layout): _, table = _s3_tables() tables = {name: table for name in ['a', 'b', 'c', 'isolated']} edges = [ @@ -263,9 +289,33 @@ def test_visualize_layout_is_deterministic_with_cycles_and_isolates(): Edge('c', 'user_id', 'a'), Edge('a', 'event_id', 'b') ] - svg = Graph(tables, edges).visualize() - assert svg == Graph(dict(reversed(list(tables.items()))), - list(reversed(edges))).visualize() + if ambiguous: + tables = { + name: + replace( + table, definition=replace( + table.definition, pkey=key, cols=[ + *table.definition.cols, + Column(key, Stype.ID, Dtype.int64) + ])) + for name, key in [('a', 'b.c'), ('a.b', 'c')] + } | { + 'c': table, + 'isolated': table + } + edges = [ + Edge('c', 'user_id', 'a.b'), + Edge('c', 'event_id', 'a'), + Edge('a.b', 'user_id', 'c'), + Edge('a', 'event_id', 'c') + ] + svg = Graph(tables, edges).visualize(layout=layout) + other = Graph(dict(reversed(list(tables.items()))), + list(reversed(edges))).visualize(layout=layout) + assert re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', + svg) == re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', other) + ids = lambda value: set(re.findall(r' id="([^"]+)"', value)) + assert ids(svg).isdisjoint(ids(other)) root = ElementTree.fromstring(svg) text = ''.join(root.itertext()) assert all(name in text for name in tables) @@ -274,11 +324,19 @@ def test_visualize_layout_is_deterministic_with_cycles_and_isolates(): element.text for element in root.iter('{http://www.w3.org/2000/svg}title') } - assert all( - f'{edge.src_table}.{edge.fkey} → {edge.dst_table}.event_id' in titles - for edge in edges) + assert all(f'{edge.src_table}.{edge.fkey} → {edge.dst_table}.' + f'{tables[edge.dst_table].definition.pkey}' in titles + for edge in edges) paths = root.findall('.//{http://www.w3.org/2000/svg}path[@fill="none"]') assert len(paths) == len(edges) + assert { + tuple( + path.get(f'data-{part}') + for part in ('src-table', 'src-column', 'dst-table', 'dst-column')) + for path in paths + } == {(edge.src_table, edge.fkey, edge.dst_table, + tables[edge.dst_table].definition.pkey) + for edge in edges} assert len({path.attrib['d'] for path in paths}) == len(edges) assert any(re.search('[CQ]', path.attrib['d']) for path in paths) @@ -323,3 +381,177 @@ def test_visualize_rejects_invalid_metadata(case, message): def test_visualize_empty_graph(): svg = Graph([]).visualize() assert 'No tables' in ''.join(ElementTree.fromstring(svg).itertext()) + + +@pytest.mark.parametrize('layout', + [None, '', 'flow', 'Original', 'circular', [], 1]) +def test_visualize_rejects_invalid_layout_before_source_access(layout): + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + side_effect=AssertionError('source accessed')): + with pytest.raises(ValueError, match='layout must be'): + Graph(_s3_tables()).visualize(layout=layout, sample_rows=3) + + +def test_visualize_layouts_preserve_schema_and_export_selection(tmp_path): + graph = Graph(_s3_tables(), [Edge('events', 'user_id', 'users')]) + schemas = [] + svg_ns = '{http://www.w3.org/2000/svg}' + for layout in ('original', 'compact'): + destination = tmp_path / f'{layout}.svg' + with patch.object(S3Connector, 'source_backend', + new_callable=PropertyMock, + side_effect=AssertionError('source accessed')): + svg = graph.visualize(destination, layout=layout, show_cols=True) + assert destination.read_text() == svg + root = ElementTree.fromstring(svg) + assert not root.findall(f'.//{svg_ns}script') + cards = root.findall(f'{svg_ns}g[@data-kumo-card]') + paths = root.findall(f'.//{svg_ns}path[@class="kumo-edge"]') + schemas.append(( + [card.get('data-kumo-card') for card in cards], + [''.join(card.itertext()) for card in cards], + [tuple(path.get(f'data-{part}') for part in + ('src-table', 'src-column', 'dst-table', 'dst-column')) + for path in paths], + )) + assert schemas[0] == schemas[1] + + +@pytest.mark.parametrize('sample_rows', [-1, 4, 1.5, True, None]) +def test_visualize_rejects_invalid_sample_count(sample_rows): + with pytest.raises(ValueError, match='sample_rows'): + Graph(_s3_tables()).visualize(sample_rows=sample_rows) + + +@pytest.mark.parametrize( + 'outcome', ['values', 'empty', 'unavailable', 'error', 'sql', 'collision', + 'nulls']) +def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): + import pandas as pd + + tables = _s3_tables() + sql = 'SELECT id FROM users' if outcome == 'sql' else None + if sql: + tables[0] = replace( + tables[0], definition=replace( + tables[0].definition, source_table=CustomSQLSourceTable( + connector_id=tables[0].connector.id, table='users', + sql=sql))) + secret = '' + frame = pd.DataFrame({ + 'id': [secret, None, ['alpha', 'beta'], 'later non-null'], + 'event_id': ['x' * 1000, None, 'third', 'later non-null'], + 'user_id': [1, 2, 3, 4], + }) + if outcome == 'nulls': + frame = pd.DataFrame({ + 'id': [None, pd.NA, pd.NaT], + 'event_id': [float('nan')] * 3, + 'user_id': [None, 7, None], + }) + if outcome == 'empty': + frame = frame.iloc[:0] + backend = Mock() + backend.get_sample.return_value.sample_rows = ( + None if outcome == 'unavailable' else frame.to_json(orient='table')) + if outcome == 'error': + backend.get_sample.side_effect = RuntimeError('credential secret') + if outcome == 'collision': + tables = [ + replace( + table, table_name=name, + definition=replace(table.definition, + cols=[Column(key, Stype.ID, + Dtype.int64)], pkey=key)) + for table, name, key in zip(tables, ['a', 'a.b'], ['b.c', 'c']) + ] + backend.get_sample.side_effect = lambda connector, name, *args, **kw: Mock( + sample_rows=pd.DataFrame({ + 'b.c' if name == 'a' else 'c': [name] + }).to_json(orient='table')) + destination = tmp_path / 'graph.svg' + with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, + return_value=backend), patch.object( + Graph, 'snapshot', side_effect=AssertionError): + svg = Graph(tables).visualize(destination, sample_rows=3, + show_cols=True) + html = svg._repr_html_() + repeat = svg._repr_html_() + assert re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', + html) == re.sub(r'kumo-[0-9a-f]{32}', 'kumo-ID', repeat) + assert set(re.findall(r' id="([^"]+)"', html)).isdisjoint( + re.findall(r' id="([^"]+)"', repeat)) + assert backend.get_sample.call_count == 2 + for call, table in zip(backend.get_sample.call_args_list, + sorted(tables, key=lambda table: table.table_name)): + assert call.args == (table.connector, table.table_name, 30) + assert call.kwargs == { + 'sql': sql if table.table_name == 'users' else None, + 'timeout': 10 + } + assert 'Sample values' not in svg + assert svg._repr_svg_() == destination.read_text() == str(svg) + assert 'credential secret' not in html + assert '') + + _HOVER + _CAMERA) + return render_widget(body, style=_STYLE, script=script, + data={'layouts': layouts, 'initial': initial_layout}, + class_name='kumo-graph', + attributes={'data-highlight': mode}) diff --git a/packages/kumo-engine/kumo/engine/_widget.py b/packages/kumo-engine/kumo/engine/_widget.py new file mode 100644 index 000000000..bc60f590b --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_widget.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Private, self-contained notebook presentation and per-output lifetime.""" + +import json +from html import escape +from string import Template +from types import MappingProxyType +from uuid import uuid4 + +THEME = MappingProxyType({ + 'font': 'Arial, Helvetica, sans-serif', + 'foreground': '#202020', 'muted': '#64748b', 'accent': '#386000', + 'header_background': '#f4f8ec', 'header_border': '#c8d9ad', + 'tooltip_header_border': '#e1e9d6', 'grid': '#d9ddd5', + 'edge': '#a1a1a1', 'focus': '#598d13', 'card_border': '#94a3b8', + 'chrome_hover': '#e8e8e8', 'chrome_foreground': '#555', + 'chrome_shadow': '0 1px 4px #0000001a', 'background': 'white', +}) + +_STYLE = Template(r""" +.kumo-widget{position:relative;display:flex;flex-direction:column; +max-width:100%;padding:8px;box-sizing:border-box;background:$background; +color:$foreground;font:14px $font} +.kumo-widget [hidden]{display:none!important} +:where(.kumo-widget-toolbar){display:flex;align-items:center;gap:6px; +flex:none;margin-bottom:16px} +:where(.kumo-widget) .kumo-widget-button{box-sizing:border-box;display:inline-flex; +align-items:center;justify-content:center;flex:0 0 32px; +width:32px!important;min-width:32px!important;max-width:32px!important; +height:32px!important;min-height:32px!important;max-height:32px!important; +padding:6px;margin:0;background:$background;color:$chrome_foreground; +border:0;border-radius:16px;cursor:pointer;font:18px/18px $font} +.kumo-widget-toolbar>.kumo-widget-button{box-shadow:$chrome_shadow} +:where(.kumo-widget) .kumo-widget-button:hover{background:$chrome_hover} +:where(.kumo-widget) .kumo-widget-button:focus-visible{ +outline:2px solid $focus;outline-offset:2px} +:where(.kumo-widget) .kumo-widget-button>svg{display:block;flex:none; +width:18px!important;height:18px!important} +""").substitute(THEME) + + +def button(cls: str, label: str, icon: str = '', text: str = '') -> str: + """Render a disabled, labeled control for feature-owned initialization.""" + graphic = (f'' + if icon else escape(text)) + label = escape(label) + return (f'') + + +def render_widget(body: str, *, style: str = '', script: str = '', + data: dict | None = None, class_name: str = '', + attributes: dict[str, str] | None = None) -> str: + """Compose trusted feature markup/code with escaped data and local cleanup. + + Feature scripts use host, widgetId, data, signal and onDispose(callback). + Root attributes are data-* or aria-* names; controls are enabled by features. + """ + attributes = attributes or {} + if any(not name.startswith(('data-', 'aria-')) or not all( + c.isascii() and (c.isalnum() or c == '-') for c in name) + for name in attributes): + raise ValueError('Widget attributes must be data-* or aria-* names.') + attrs = ''.join(f' {key}="{escape(value)}"' + for key, value in attributes.items()) + payload = json.dumps(data or {}, ensure_ascii=True, separators=(',', ':')) + for char, encoded in [('<', r'\u003c'), ('>', r'\u003e'), ('&', r'\u0026')]: + payload = payload.replace(char, encoded) + return ( + f'
' + f'{body}' + f'
') diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 13c44ec59..fc92194f9 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -2,6 +2,7 @@ # All rights reserved. # SPDX-License-Identifier: Apache-2.0 +import json import re from dataclasses import replace from pathlib import Path @@ -409,6 +410,35 @@ def test_visualize_empty_graph(): assert 'No tables' in ''.join(ElementTree.fromstring(svg).itertext()) +def test_widget_shell_composes_non_graph_content_and_escapes_data(): + from kumo.engine._widget import button, render_widget + + label = ' & values' + body = ('
' + + button('chart-toggle', label, text='') + '
' + '' + '') + rendered = [ + render_widget(body, data={'labels': [label], 'values': [0.75]}, + attributes={'aria-label': label}) for _ in range(2) + ] + roots = [ElementTree.fromstring(html) for html in rendered] + assert roots[0].get('id') != roots[1].get('id') + with pytest.raises(ValueError, match='data-.*aria-'): + render_widget(body, attributes={'onclick': 'alert(1)'}) + for root, html in zip(roots, rendered): + assert root.get('aria-label') == label + control = root.find('.//button') + assert control.get('aria-label') == label + assert control.text == '' + assert 'disabled' in control.attrib + payload = root.find('.//script[@type="application/json"]') + assert json.loads(payload.text) == {'labels': [label], 'values': [0.75]} + assert root.find('.//{http://www.w3.org/2000/svg}rect').get('width') == '75' + assert '' frame = pd.DataFrame({ - 'id': [secret, None, ['alpha', 'beta'], 'later non-null'], - 'event_id': ['x' * 1000, None, 'third', 'later non-null'], - 'user_id': [1, 2, 3, 4], + 'id': [secret, None, ['alpha', 'beta'], 'later non-null', + 'fourth', 'fifth', 'sixth', 'outside cap'], + 'event_id': ['x' * 1000, None, 'third', 'later non-null', + 'fourth', 'fifth', 'sixth', 'outside cap'], + 'user_id': list(range(1, 9)), }) if outcome == 'nulls': frame = pd.DataFrame({ @@ -514,7 +518,7 @@ def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, return_value=backend), patch.object( Graph, 'snapshot', side_effect=AssertionError): - svg = Graph(tables).visualize(destination, sample_rows=3, + svg = Graph(tables).visualize(destination, show_samples=True, show_cols=True) html = svg._repr_html_() if outcome == 'values': @@ -539,7 +543,12 @@ def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): if outcome in ('values', 'sql'): assert secret in details and '\nnull' not in details assert "['alpha', 'beta']" in details - assert 'later non-null' in details + assert 'later non-null' in details and 'sixth' in details + assert 'outside cap' not in details + for tip in tips: + if 'Sample values:\n' in tip.get('data-kumo-tip'): + assert len(tip.get('data-kumo-tip').split( + 'Sample values:\n')[1].splitlines()) == 6 assert 'x' * 121 not in details elif outcome == 'collision': rows = [tip for tip in tips if tip.get('data-kumo-row')] @@ -558,8 +567,7 @@ def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): assert 'Sample values: unavailable' in details -@pytest.mark.parametrize('count', [1, 3]) -def test_visualize_samples_bound_preview_and_select_columns_independently(count): +def test_visualize_samples_bound_preview_and_select_columns_independently(): import pandas as pd from kumo.engine._visualization import _preview @@ -568,15 +576,16 @@ def test_visualize_samples_bound_preview_and_select_columns_independently(count) backend = Mock() backend.get_sample.return_value.sample_rows = pd.DataFrame({ 'id': [None] * 29 + ['last allowed', 'outside preview'], - 'other': ['first', None, 'second', 'third'] + [None] * 27, + 'other': ['first', None, 'second', 'third', 'fourth', 'fifth', + 'sixth', 'outside cap'] + [None] * 23, }).to_json(orient='table') with patch.object(S3Connector, 'source_backend', new_callable=PropertyMock, return_value=backend): - preview = _preview(table, count, ['id', 'other']) + preview = _preview(table, ['id', 'other']) assert preview['id'] == "\nSample values:\n'last allowed'" assert preview['other'].splitlines()[2:] == [ - "'first'", "'second'", "'third'" - ][:count] + "'first'", "'second'", "'third'", "'fourth'", "'fifth'", "'sixth'" + ] @pytest.mark.parametrize('count', [30, 31]) @@ -587,7 +596,7 @@ def test_visualize_table_cap_precedes_source_access(count): new_callable=PropertyMock) as backend: if count == 31: with pytest.raises(ValueError, match='at most 30 tables'): - graph.visualize(sample_rows=3) + graph.visualize(show_samples=True) else: root = ElementTree.fromstring(graph.visualize()) assert len(root.findall( From d92abbc2404166b54fbad6b76ffdbcef24118a23 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Thu, 29 Oct 2026 02:30:05 +0000 Subject: [PATCH 22/39] Format graph timestamp samples as plain date and time --- packages/kumo-engine/kumo/engine/_visualization.py | 4 +++- packages/kumo-engine/test/test_graph_validation.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index 0bf8783c2..f227b0204 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -4,6 +4,7 @@ """SVG rendering of declared table relationships.""" import json +from datetime import date from html import escape from reprlib import Repr from typing import Mapping, Sequence @@ -130,7 +131,8 @@ def _preview(table: Table, columns: Sequence[str]) -> dict[str, str]: or isinstance(value, float) and value != value) if null: continue - label = short.repr(value) + label = (str(value) if isinstance(value, date) else + short.repr(value)) labels.append(label if len(label) <= 160 else label[:157] + '...') if len(labels) == 6: diff --git a/packages/kumo-engine/test/test_graph_validation.py b/packages/kumo-engine/test/test_graph_validation.py index 20da2f4d1..a7300453e 100644 --- a/packages/kumo-engine/test/test_graph_validation.py +++ b/packages/kumo-engine/test/test_graph_validation.py @@ -481,12 +481,20 @@ def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): connector_id=tables[0].connector.id, table='users', sql=sql))) secret = '' + definition = tables[0].definition + assert definition is not None + tables[0] = replace(tables[0], definition=replace( + definition, cols=[ + *definition.cols, + Column('created_at', Stype.timestamp, Dtype.date)])) frame = pd.DataFrame({ 'id': [secret, None, ['alpha', 'beta'], 'later non-null', 'fourth', 'fifth', 'sixth', 'outside cap'], 'event_id': ['x' * 1000, None, 'third', 'later non-null', 'fourth', 'fifth', 'sixth', 'outside cap'], 'user_id': list(range(1, 9)), + 'created_at': pd.date_range('2026-10-01 09:00:00.123', periods=8, + tz='UTC'), }) if outcome == 'nulls': frame = pd.DataFrame({ @@ -541,6 +549,8 @@ def test_visualize_samples_once_and_keeps_svg_metadata_only(tmp_path, outcome): './/{http://www.w3.org/2000/svg}g[@data-kumo-tip]') details = '\n'.join(tip.get('data-kumo-tip') for tip in tips) if outcome in ('values', 'sql'): + assert '2026-10-01 09:00:00.123000+00:00' in details + assert 'Timestamp(' not in details assert secret in details and '\nnull' not in details assert "['alpha', 'beta']" in details assert 'later non-null' in details and 'sixth' in details From 3ea920e552f056039e1ff96ff92c8fe936c18058 Mon Sep 17 00:00:00 2001 From: devanj08 Date: Thu, 29 Oct 2026 15:40:21 +0000 Subject: [PATCH 23/39] Add graph PDF export and deliberate hover navigation --- packages/kumo-engine/README.md | 11 +- .../kumo-engine/kumo/engine/_visualization.py | 28 ++- .../kumo/engine/_visualization_pdf.py | 222 ++++++++++++++++++ .../kumo/engine/_visualization_viewer.py | 172 ++++++++------ packages/kumo-engine/kumo/engine/graph.py | 17 +- packages/kumo-engine/pyproject.toml | 1 + packages/kumo-engine/setup.py | 2 + .../kumo-engine/test/test_graph_validation.py | 42 +++- pyproject.toml | 3 +- uv.lock | 211 +++++++++++++++++ 10 files changed, 626 insertions(+), 83 deletions(-) create mode 100644 packages/kumo-engine/kumo/engine/_visualization_pdf.py diff --git a/packages/kumo-engine/README.md b/packages/kumo-engine/README.md index 13c05982c..e133c5451 100644 --- a/packages/kumo-engine/README.md +++ b/packages/kumo-engine/README.md @@ -102,15 +102,16 @@ To use your own data, point `FileConnector` at a directory of CSV or Parquet fil ## Visualize table relationships -Display declared tables and relationships in a notebook, or export SVG: +Display declared tables and relationships in a notebook, or export SVG or PDF: ```python graph.visualize() graph.visualize("relationships.svg", show_cols=True) +graph.visualize("relationships.pdf", show_cols=True, layout="compact") graph.visualize(layout="compact") ``` -Visualization reads no source data by default and needs no additional rendering package. +Visualization reads no source data by default; rendering dependencies are installed with Kumo Engine. The return value is SVG text with notebook display support. The default view includes primary keys, relationship foreign keys, and the declared time column (`TS`). `show_cols=True` adds other columns up to 10 rows per table, or 5 above 10 tables; key and time columns always remain visible. @@ -121,12 +122,16 @@ Drag to pan, use the controls or Ctrl/Cmd+wheel to zoom, and choose Fit to reset Hover or focus a header or column for details. Interactive output requires trusted HTML; fullscreen also needs permission from the notebook host. SVG exports contain the initial arrangement and native tooltips. +Use the download button to save the entire selected layout as a PDF on your computer. +The PDF includes a paginated index of all declared tables, columns, data types, key/time roles, and connections, including columns omitted from the diagram. +Passing a `.pdf` path to `visualize` saves the same document on the filesystem where Kumo Engine runs, using the requested initial layout; it overwrites an existing file. +PDF text uses embedded DejaVu fonts and standard CJK font fallbacks; unsupported characters are shown as explicit Unicode escapes. `show_samples=True` shows up to six non-null examples per visible column in two columns and up to three rows, sampled independently from up to 30 preview rows per table. The default `show_samples=False` hides samples and reads no source data. Previews can run warehouse queries and incur costs; the row limit does not bound bytes scanned. Unavailable or empty previews are labeled. -Sample values are saved in notebook HTML, but never included in SVG text or exports. +Sample values are saved in notebook HTML, but never included in SVG or PDF exports. ### Developing another notebook widget diff --git a/packages/kumo-engine/kumo/engine/_visualization.py b/packages/kumo-engine/kumo/engine/_visualization.py index f227b0204..1914af924 100644 --- a/packages/kumo-engine/kumo/engine/_visualization.py +++ b/packages/kumo-engine/kumo/engine/_visualization.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 """SVG rendering of declared table relationships.""" +import base64 import json from datetime import date from html import escape @@ -27,17 +28,27 @@ class SVG(str): def __new__(cls, value: str, samples: dict[str, str] | None = None, namespace: str = '', layouts: dict | None = None, - initial_layout: str = 'standard'): + initial_layout: str = 'standard', index: list | None = None): obj = super().__new__(cls, value) obj._samples = samples or {} obj._namespace = namespace obj._layouts = layouts or {} obj._initial_layout = initial_layout + obj._index = index or [] + obj._pdfs = {} return obj def _repr_svg_(self) -> str: return str(self) + def _pdf(self, layout: str) -> bytes: + from kumo.engine._visualization_pdf import render_pdf + + if layout not in self._pdfs: + self._pdfs[layout] = render_pdf(self, self._layouts[layout], + self._index) + return self._pdfs[layout] + def _repr_html_(self) -> str: from kumo.engine._visualization_viewer import render_html @@ -46,7 +57,9 @@ def _repr_html_(self) -> str: svg = svg.replace(metadata, sampled) if self._namespace: svg = svg.replace(self._namespace, 'kumo-' + uuid4().hex) - return render_html(svg, self._layouts, self._initial_layout) + pdfs = {name: base64.b64encode(self._pdf(name)).decode('ascii') + for name in self._layouts} + return render_html(svg, self._layouts, self._initial_layout, pdfs) def _wrap(text: str, width: int) -> list[str]: @@ -464,5 +477,14 @@ def render(tables: Mapping[str, Table], edges: Sequence[Edge], *, svg.append('') for x, y in geometry['handles']: svg.append(f'') + index = [dict( + name=name, + columns=[dict(name=col.name, type=col.dtype.value, + role=keys[name].get(col.name, '')) + for col in definitions[name].cols], + connections=[f'{edge.src_table}.{edge.fkey} → ' + f'{edge.dst_table}.{definitions[edge.dst_table].pkey}' + for edge in edges if edge.src_table == name], + ) for name in names] return SVG(''.join(svg) + '', samples, uid, layouts=layouts, - initial_layout=selected) + initial_layout=selected, index=index) diff --git a/packages/kumo-engine/kumo/engine/_visualization_pdf.py b/packages/kumo-engine/kumo/engine/_visualization_pdf.py new file mode 100644 index 000000000..a35034ccc --- /dev/null +++ b/packages/kumo-engine/kumo/engine/_visualization_pdf.py @@ -0,0 +1,222 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. +# All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Vector graph diagrams with a paginated schema and relationship index.""" + +from functools import lru_cache +from html import escape +from importlib.resources import files +from io import BytesIO +from itertools import groupby + +from lxml import etree as ET +from reportlab.graphics import renderPDF +from reportlab.lib import colors +from reportlab.lib.pagesizes import A4 +from reportlab.lib.styles import ParagraphStyle +from reportlab.pdfbase import pdfmetrics +from reportlab.pdfbase.cidfonts import UnicodeCIDFont +from reportlab.pdfbase.ttfonts import TTFont +from reportlab.platypus import ( + BaseDocTemplate, + Frame, + LongTable, + NextPageTemplate, + PageBreak, + PageTemplate, + Paragraph, + Spacer, +) +from svglib.fonts import FontMap +from svglib.svglib import svg2rlg + +from kumo.engine._widget import THEME + + +class _PDFFontMap(FontMap): + def find_font(self, font_name, weight='normal', style='normal'): + if font_name == 'KumoPDF': + return ('KumoPDF-Bold' if weight in ('bold', '700') + else 'KumoPDF'), True + if font_name in ('STSong-Light', 'HeiseiKakuGo-W5', 'HYSMyeongJo-Medium'): + return font_name, True + return super().find_font(font_name, weight, style) + + +@lru_cache(maxsize=1) +def _fonts(): + directory = files('matplotlib').joinpath('mpl-data/fonts/ttf') + for suffix in ('', '-Bold'): + name = 'KumoPDF' + suffix + font = TTFont(name, str(directory.joinpath(f'DejaVuSans{suffix}.ttf'))) + pdfmetrics.registerFont(font) + for name in ('STSong-Light', 'HeiseiKakuGo-W5', 'HYSMyeongJo-Medium'): + pdfmetrics.registerFont(UnicodeCIDFont(name)) + pdfmetrics.registerFontFamily('KumoPDF', normal='KumoPDF', + bold='KumoPDF-Bold', italic='KumoPDF', + boldItalic='KumoPDF-Bold') + return _PDFFontMap(), pdfmetrics.getFont('KumoPDF').face.charToGlyph + + +def _font_runs(text): + supported = _fonts()[1] + + def font(char): + code = ord(char) + if code in supported or char in '\n\t': + return 'KumoPDF' + if 0x3400 <= code <= 0x9FFF: + return 'STSong-Light' + if 0x3000 <= code <= 0x33FF or 0xFF00 <= code <= 0xFFEF: + return 'HeiseiKakuGo-W5' + if 0xAC00 <= code <= 0xD7AF: + return 'HYSMyeongJo-Medium' + return None + + for name, chars in groupby(text, font): + value = ''.join(chars) + yield name or 'KumoPDF', (value if name else + value.encode('unicode_escape').decode('ascii')) + + +def render_pdf(svg: str, geometry: dict, index: list[dict]) -> bytes: + """Export an entire layout and every declared column without sample values.""" + font_map, _ = _fonts() + escaped = False + root = ET.fromstring(svg.encode('utf-8')) + ns = '{http://www.w3.org/2000/svg}' + width, height = geometry['width'], geometry['height'] + root.set('width', str(width)) + root.set('height', str(height)) + root.set('viewBox', f'0 0 {width} {height}') + root.set('font-family', 'KumoPDF') + for child in list(root): + if child.tag in (ns + 'defs', ns + 'circle') or ( + child.tag == ns + 'rect' and + child.get('fill', '').startswith('url(')): + root.remove(child) + elif child.tag == ns + 'rect': + child.set('width', str(width)) + child.set('height', str(height)) + elif 'data-kumo-card' in child.attrib: + x, y = geometry['positions'][child.get('data-kumo-card')] + child.set('transform', f'translate({x} {y})') + for path, value in zip(root.findall(f'.//{ns}path[@class="kumo-edge"]'), + geometry['paths']): + path.set('d', value) + for x, y in geometry['handles']: + ET.SubElement(root, ns + 'circle', cx=str(x), cy=str(y), r='4', + fill=THEME['muted']) + for element in root.iter(): + if element.get('fill') == 'transparent': + element.set('fill', 'none') + for element in root.iter(ns + 'text'): + original = element.text or '' + runs = list(_font_runs(original)) + count = sum(len(text) for _, text in runs) + size = float(element.get('font-size', 16)) + if count > len(original): + escaped = True + size *= len(original) / count + element.set('font-size', str(size)) + widths = [pdfmetrics.stringWidth( + text, 'KumoPDF-Bold' if font == 'KumoPDF' and + element.get('font-weight') == '700' else font, size) + for font, text in runs] + x = float(element.get('x', 0)) + if element.get('text-anchor') == 'end': + x -= sum(widths) + element.set('text-anchor', 'start') + element.text = None + for (font, text), run_width in zip(runs, widths): + ET.SubElement(element, ns + 'tspan', { + 'font-family': font, 'x': str(x)}).text = text + x += run_width + drawing = svg2rlg(BytesIO(ET.tostring(root)), font_map=font_map) + if drawing is None: + raise ValueError('Could not render the graph as PDF.') + graph_size = (min(14400, max(842, drawing.width + 64)), + min(14400, max(595, drawing.height + 96))) + output = BytesIO() + document = BaseDocTemplate(output, pagesize=graph_size, pageCompression=1, + title='Kumo graph and schema index', + author='Kumo Engine') + + def graph_page(canvas, doc): + canvas.setFont('KumoPDF-Bold', 16) + canvas.setFillColor(colors.HexColor(THEME['accent'])) + canvas.drawString(32, graph_size[1] - 32, 'Table relationships') + scale = min(1, (graph_size[0] - 64) / drawing.width, + (graph_size[1] - 96) / drawing.height) + canvas.saveState() + canvas.translate((graph_size[0] - drawing.width * scale) / 2, + (graph_size[1] - drawing.height * scale) / 2 - 8) + canvas.scale(scale, scale) + renderPDF.draw(drawing, canvas, 0, 0) + canvas.restoreState() + canvas.setFont('KumoPDF', 9) + canvas.drawString(32, 20, 'PK: primary key FK: foreign key TS: time column') + + def index_page(canvas, doc): + canvas.setFont('KumoPDF', 9) + canvas.setFillColor(colors.HexColor(THEME['muted'])) + canvas.drawRightString(A4[0] - 36, 20, f'Page {doc.page}') + + document.addPageTemplates([ + PageTemplate('graph', [Frame(32, 32, graph_size[0] - 64, + graph_size[1] - 64)], onPage=graph_page, + pagesize=graph_size), + PageTemplate('index', [Frame(30, 30, A4[0] - 60, A4[1] - 60)], + onPage=index_page, pagesize=A4), + ]) + normal = ParagraphStyle('body', fontName='KumoPDF', fontSize=9, leading=13, + textColor=colors.HexColor(THEME['foreground']), + wordWrap='CJK') + heading = ParagraphStyle('heading', parent=normal, fontName='KumoPDF-Bold', + fontSize=14, leading=19, spaceBefore=12, + spaceAfter=8, keepWithNext=True) + + def paragraph(value, style=normal): + nonlocal escaped + runs = list(_font_runs(value)) + escaped |= ''.join(text for _, text in runs) != value + return Paragraph(''.join( + f'' + f'{escape(text).replace(" ", " ").replace(chr(10), "
")}
' + for font, text in runs), style) + + story = [Spacer(1, 1), NextPageTemplate('index'), PageBreak(), + paragraph('Tables and connections', heading), + paragraph('All declared columns are listed, including columns ' + 'omitted from the diagram. Sample values are excluded.')] + for table in index: + story.append(paragraph(table['name'], heading)) + rows = [[paragraph(value) for value in ('Column', 'Data type', 'Role')]] + rows.extend([paragraph(col['name']), paragraph(col['type']), + paragraph(col['role'] or 'Column')] + for col in table['columns']) + if table['columns']: + story.append(LongTable( + rows, colWidths=[285, 140, A4[0] - 72 - 425], repeatRows=1, + splitInRow=1, hAlign='LEFT', style=[ + ('BACKGROUND', (0, 0), (-1, 0), + colors.HexColor(THEME['header_background'])), + ('VALIGN', (0, 0), (-1, -1), 'TOP'), + ('BOTTOMPADDING', (0, 0), (-1, -1), 6), + ('TOPPADDING', (0, 0), (-1, -1), 6), + ('LINEBELOW', (0, 0), (-1, -1), .3, + colors.HexColor(THEME['card_border'])), + ])) + else: + story.append(paragraph('No columns')) + story.append(Spacer(1, 6)) + story.extend(paragraph(connection) for connection in + table['connections'] or ['No outgoing connections']) + if not index: + story.append(paragraph('No tables')) + if escaped: + story.insert(5, paragraph( + 'Characters unavailable in the PDF fonts use Unicode escapes ' + '(\\uXXXX or \\UXXXXXXXX).')) + document.build(story) + return output.getvalue() diff --git a/packages/kumo-engine/kumo/engine/_visualization_viewer.py b/packages/kumo-engine/kumo/engine/_visualization_viewer.py index 6bb00c68c..d7c04faba 100644 --- a/packages/kumo-engine/kumo/engine/_visualization_viewer.py +++ b/packages/kumo-engine/kumo/engine/_visualization_viewer.py @@ -13,7 +13,7 @@ .kumo-graph:fullscreen{width:100%;height:100%;max-height:none} .kumo-graph-toolbar{display:grid; grid-template-columns:minmax(0,1fr) auto minmax(0,1fr)} -.kumo-graph-layout{display:flex;grid-column:2;justify-self:center; +.kumo-graph-layout{display:flex;grid-column:2;grid-row:1;justify-self:center; border-radius:18px;background:$background;padding:2px;color:$chrome_foreground; font:13px $font} .kumo-graph-layout label{position:relative;cursor:pointer} @@ -22,14 +22,15 @@ padding:0 12px;border-radius:16px} .kumo-graph-layout input:checked+span{background:$chrome_hover;color:$foreground} .kumo-graph-actions{display:flex;align-items:center;gap:6px; -grid-column:3;justify-self:end} +grid-column:1;grid-row:1;justify-self:start} .kumo-graph-zoom{display:flex;align-items:center;gap:0;flex:none; background:$background;border-radius:18px} .kumo-graph-layout,.kumo-graph-zoom,.kumo-graph-actions>button{ box-shadow:$chrome_shadow} -@container kumo-graph (max-width:560px){ -.kumo-graph-toolbar{grid-template-columns:minmax(0,1fr)} -.kumo-graph-layout,.kumo-graph-actions{grid-column:1}} +@container kumo-graph (max-width:900px){ +.kumo-graph-toolbar{grid-template-columns:minmax(0,1fr);row-gap:12px} +.kumo-graph-layout{grid-column:1;grid-row:2}} +@container kumo-graph (max-width:340px){.kumo-graph-actions{gap:4px}} .kumo-graph-layout input:focus-visible+span{ outline:2px solid $focus;outline-offset:2px} .kumo-graph-scale{min-width:44px;text-align:center;color:#666; @@ -109,11 +110,15 @@ tip.className='kumo-graph-tooltip';tip.role='tooltip'; tip.id='kumo-tip-'+Math.random().toString(36).slice(2);host.append(tip); let active=null, card=null, timer, placement=null, hoverLock=null, pointer=null; +let panTimer=null,panFrame=0,dwellPoint=null; +const HOVER_DWELL=350,PAN_DURATION=250; const firstCard=svg.querySelector(':scope>[data-kumo-card]'); const element=(cls,text)=>{const n=document.createElement('div'); n.className='kumo-tip-'+cls;n.textContent=text;return n}; const cancelHide=()=>{clearTimeout(timer);timer=null}; -const hide=()=>{cancelHide();tip.removeAttribute('data-open'); +const cancelPan=()=>{clearTimeout(panTimer);panTimer=null; +cancelAnimationFrame(panFrame);panFrame=0}; +const hide=()=>{cancelHide();cancelPan();tip.removeAttribute('data-open'); placement=hoverLock=null; if(host.classList.contains('kumo-links-focused')) edges.forEach(edge=>svg.insertBefore(edge.parentElement,firstCard)); @@ -231,68 +236,53 @@ for(const curve of cached.curves)visit(curve.map(([x,y])=>({ x:matrix.a*x+matrix.c*y+matrix.e,y:matrix.b*x+matrix.d*y+matrix.f}))); return left===Infinity?null:[left-2,right+2]}; -const classify=()=>{ -const b=viewport(),c=card.getBoundingClientRect(),table=active.dataset.kumoTable; -const visible=Array.from(svg.querySelectorAll(':scope>[data-kumo-card]'), -x=>x.getBoundingClientRect()).filter(r=>intersects(r,b)); -const leftmost=c.left<=Math.min(...visible.map(r=>r.left))+1; -const rightmost=c.right>=Math.max(...visible.map(r=>r.right))-1; -const role=active.dataset.kumoTip.split('\n').find(x=>x.startsWith('Role: '))||''; -const pk=/\bPK\b/.test(role),fk=/\bFK\b/.test(role); -const incident=edges.filter(e=>e.dataset.srcTable===table||e.dataset.dstTable===table); -const selected=incident.filter(e=>e.classList.contains('kumo-edge-active')); -const relevant=active.dataset.kumoRow?selected:incident; -const extents=new Map(),extent=e=>{if(!extents.has(e))extents.set(e,edgeExtent(e,b));return extents.get(e)}; -let side; -if(leftmost!==rightmost)side=leftmost?'left':'right'; -else if(pk!==fk)side=pk?'left':'right'; -else{ -let left=0,right=0;for(const e of relevant){const r=extent(e);if(!r)continue; -if(r[0]c.right+1)right++} -side=left===right?(c.left-b.left>=b.right-c.right?'left':'right'): -(left{ -if(!active||!placement)return; +if(!active)return; const b=viewport(),a=active.getBoundingClientRect(),c=card.getBoundingClientRect(); if(b.right<=b.left||b.bottom<=b.top){hide();return} tip.style.maxWidth=Math.max(0,Math.min(320,(b.right-b.left)/b.sx))+'px'; tip.style.maxHeight=Math.max(0,(b.bottom-b.top)/b.sy)+'px'; layoutConnections(b.sx); -const t=tip.getBoundingClientRect(),left=placement.side==='left'; -let x=(left?c.left-t.width-12*b.sx:c.right+12*b.sx)+placement.offset*b.sx; -const y=Math.max(b.top,Math.min(a.top,b.bottom-t.height)); -if(allowPan===true){ -const clamp=(v,lo,hi)=>Math.max(lo,Math.min(v,hi)); +const t=tip.getBoundingClientRect(),clamp=(v,lo,hi)=>Math.max(lo,Math.min(v,hi)); +const y=clamp(a.top,b.top,b.bottom-t.height); +if(!placement){ const dy=clamp(y-a.top,b.top+Math.min(8*b.sy,a.height/2)-a.bottom, b.bottom-Math.min(8*b.sy,a.height/2)-a.top); -if(active.dataset.kumoRow){ -// Compare the final popup with only route portions visible before the one pan. -const top=Math.max(b.top,Math.min(a.top+dy,b.bottom-t.height))-dy; +const top=clamp(a.top+dy,b.top,b.bottom-t.height)-dy; const band={...b,top:Math.max(b.top,top-2),bottom:Math.min(b.bottom,top+t.height+2)}; const spans=[]; if(band.top{ +const left=side==='left',adjacent=left?c.left-t.width-12*b.sx:c.right+12*b.sx; +let x=adjacent; spans.sort((a,z)=>left?z[1]-a[1]:a[0]-z[0]); -const adjacent=x; for(const span of spans)if(xspan[0]) x=left?span[0]-t.width-12*b.sx:span[1]+12*b.sx; -placement.offset+=(x-adjacent)/b.sx} -const dx=clamp(clamp(xb.right?b.right-x-t.width:0, --(b.right-b.left),b.right-b.left),b.left+Math.min(24*b.sx,a.width/2)-a.right, +const needed=xb.right?b.right-x-t.width:0; +const dx=clamp(clamp(needed,-(b.right-b.left),b.right-b.left), +b.left+Math.min(24*b.sx,a.width/2)-a.right, b.right-Math.min(24*b.sx,a.width/2)-a.left); -if(Math.abs(dx)>.5||Math.abs(dy)>.5){hoverLock=pointer?{...pointer}:{}; -panBy(dx/b.sx,dy/b.sy);place();return}} +return{side,offset:(x-adjacent)/b.sx,dx,dy, +cost:Math.abs(dx-needed)*1000+Math.hypot(dx,dy)}}); +placement=options[0].cost<=options[1].cost?options[0]:options[1]} +if(allowPan===true&&(Math.abs(placement.dx)>.5||Math.abs(placement.dy)>.5)){ +hoverLock=pointer?{...pointer}:{}; +panBy(placement.dx/b.sx,placement.dy/b.sy,true);return} if(!intersects(a,b)){hide();return} -const raw=x;x=Math.max(b.left,Math.min(x,b.right-t.width)); +const raw=(placement.side==='left'?c.left-t.width-12*b.sx:c.right+12*b.sx)+ +placement.offset*b.sx; +const x=clamp(raw,b.left,b.right-t.width); tip.dataset.side=placement.side;tip.dataset.clamped=String(x!==raw||y!==a.top); tip.style.left=(x-b.h.left)/b.sx+'px';tip.style.top=(y-b.h.top)/b.sy+'px'}; -const show=(el,event)=>{if(isDragging||hoverLock)return; -cancelHide();if(active===el)return; +const schedulePan=()=>{ +clearTimeout(panTimer);dwellPoint=pointer?{...pointer}:null; +panTimer=setTimeout(()=>{panTimer=null; +if(active&&!drag&&!hoverLock){placement=null;place(true)}},HOVER_DWELL)}; +const show=(el,event)=>{if(drag||hoverLock)return; +cancelHide();if(active===el){ +if(!panTimer&&placement&&(Math.abs(placement.dx)>.5||Math.abs(placement.dy)>.5))schedulePan(); +return} if(event&&Number.isFinite(event.clientX))pointer={x:event.clientX,y:event.clientY}; hide();active=el;card=el.parentElement; el.classList.add('kumo-active'); @@ -305,8 +295,10 @@ edge.classList.toggle('kumo-edge-active',src||dst); if(src||dst){svg.insertBefore(edge.parentElement,firstCard); host.classList.add('kumo-links-focused')}})} -placement=classify();tip.setAttribute('data-open','');place(true)}; -const scheduleHide=()=>{if(timer)return;timer=setTimeout(()=>{timer=null; +tip.setAttribute('data-open','');place(); +if(active&&(Math.abs(placement.dx)>.5||Math.abs(placement.dy)>.5))schedulePan()}; +const scheduleHide=()=>{if(hoverLock)return; +clearTimeout(panTimer);panTimer=null;if(timer)return;timer=setTimeout(()=>{timer=null; if(!hoverLock&&active!==document.activeElement)hide()},150)}; view.querySelectorAll('title').forEach(el=>el.remove()); view.querySelectorAll('[data-kumo-tip]').forEach(el=>{ @@ -317,16 +309,19 @@ el.addEventListener('pointerleave',scheduleHide,{signal}); el.addEventListener('focus',()=>{hoverLock=null;show(el)},{signal}); el.addEventListener('blur',scheduleHide,{signal})}); -host.addEventListener('pointerleave',()=>{hoverLock=null;scheduleHide()},{signal}); +host.addEventListener('pointerleave',()=>{cancelPan();hoverLock=null;scheduleHide()},{signal}); document.addEventListener('pointerout',e=>{ -if(!e.relatedTarget){hoverLock=null;scheduleHide()} +if(!e.relatedTarget){cancelPan();hoverLock=null;scheduleHide()} },{signal}); document.addEventListener('pointermove',e=>{ -pointer={x:e.clientX,y:e.clientY};if(isDragging||!active)return; -if(hoverLock&&Math.hypot(pointer.x-hoverLock.x,pointer.y-hoverLock.y)<=1)return; +pointer={x:e.clientX,y:e.clientY};if(drag||!active||panFrame)return; +if(hoverLock&&Math.hypot(pointer.x-hoverLock.x,pointer.y-hoverLock.y)<=4)return; hoverLock=null;const hit=document.elementFromPoint(e.clientX,e.clientY); const next=hit?.closest('[data-kumo-tip]'); -if(next&&view.contains(next)){show(next,e);return} +if(next&&view.contains(next)){ +if(next===active&&panTimer&&dwellPoint&& +Math.hypot(pointer.x-dwellPoint.x,pointer.y-dwellPoint.y)>4)schedulePan(); +show(next,e);return} scheduleHide(); },{capture:true,signal}); @@ -391,7 +386,22 @@ __GRID_COLOR__ ${.7*scale}px,transparent ${.8*scale}px)`; label.textContent=Math.round(scale*100)+'%';host.dataset.scale=String(scale); }; -const panBy=(dx,dy)=>{fitted=false;panX+=dx;panY+=dy;resizeScene()}; +const panBy=(dx,dy,animate=false)=>{ +fitted=false; +if(!animate){panX+=dx;panY+=dy;resizeScene();return} +const x=panX,y=panY,start=performance.now(); +const duration=matchMedia('(prefers-reduced-motion:reduce)').matches?0:PAN_DURATION; +const step=now=>{ +if(!active||signal.aborted){panFrame=0;return} +if(view.clientWidth!==viewWidth||view.clientHeight!==viewHeight){ +hide();resizeScene();return} +const t=duration?Math.min(1,(now-start)/duration):1,k=t*t*(3-2*t); +panX=x+dx*k;panY=y+dy*k;resizeScene();place(); +if(!active){panFrame=0;return} +if(t<1)panFrame=requestAnimationFrame(step); +else{panFrame=0;if(placement)placement.dx=placement.dy=0; +hoverLock=pointer?{...pointer}:{}}}; +panFrame=requestAnimationFrame(step)}; const fit=()=>{ hide();fitted=true;scale=fitScale(); panX=(view.clientWidth-layoutWidth*scale)/2; @@ -405,6 +415,7 @@ }; view.addEventListener('pointerdown',e=>{ if(e.button!==0||e.pointerType==='touch')return; +cancelPan(); const rect=view.getBoundingClientRect(); drag={id:e.pointerId,x:e.clientX,y:e.clientY,panX,panY, sx:rect.width/view.clientWidth,sy:rect.height/view.clientHeight}; @@ -472,8 +483,8 @@ } },{signal}); document.addEventListener('keydown',e=>{if(e.key==='Escape')hide()},{signal}); -document.addEventListener('scroll',place,{capture:true,signal}); -window.addEventListener('resize',place,{signal}); +document.addEventListener('scroll',()=>{cancelPan();placement=null;place()},{capture:true,signal}); +window.addEventListener('resize',()=>{cancelPan();placement=null;place()},{signal}); const resizeObserver=new ResizeObserver(()=>{ if(signal.aborted)return; cancelAnimationFrame(resizeFrame); @@ -491,7 +502,23 @@ '__MUTED_COLOR__', THEME['muted']) -def render_html(svg: str, layouts: dict, initial_layout: str) -> str: +_DOWNLOAD = r""" +const download=host.querySelector('.kumo-graph-download'); +const pdfURLs=new Map(); +download.addEventListener('click',()=>{ +if(!pdfURLs.has(selected)){ +const bytes=Uint8Array.from(atob(data.pdfs[selected]),char=>char.charCodeAt(0)); +pdfURLs.set(selected,URL.createObjectURL(new Blob([bytes],{type:'application/pdf'}))); +} +const link=document.createElement('a');link.href=pdfURLs.get(selected); +link.download='graph-'+selected+'.pdf';host.append(link);link.click();link.remove(); +},{signal}); +onDispose(()=>{for(const url of pdfURLs.values())URL.revokeObjectURL(url)}); +""" + + +def render_html(svg: str, layouts: dict, initial_layout: str, + pdfs: dict | None = None) -> str: """Wrap one SVG with controls and serialized, precomputed geometries.""" options = ''.join( '