From d59cd2d15e88f0d0389f590e76f638598e6c29ec Mon Sep 17 00:00:00 2001 From: Vedaant Jain Date: Tue, 17 Nov 2026 23:11:33 +0000 Subject: [PATCH] perf(kumo-pytorch): deduplicate graph edges with a packed key np.unique(axis=1) sorts opaque two-column records. When the indices are non-negative and source * (max destination + 1) + destination fits int64, deduplicate one packed int64 key per edge instead, keeping the existing first-occurrence order; other inputs use the previous path. Co-Authored-By: Claude --- .../src/kumo/pytorch/engines/diskgraph.py | 7 +++++- .../tests/test_diskgraph_graph_output.py | 22 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py index 950c73cf6..77ccf1eea 100644 --- a/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py +++ b/packages/experimental/kumo-pytorch/src/kumo/pytorch/engines/diskgraph.py @@ -2561,7 +2561,12 @@ def _first_occurrences(pairs: np.ndarray) -> np.ndarray: if pairs.shape[1] < 2: return pairs - _, first = np.unique(pairs, axis=1, return_index=True) + lowest, source_max, width = int(pairs.min()), int(pairs[0].max()), int(pairs[1].max()) + 1 + if lowest >= 0 and (source_max + 1) * width < 2**63: + source, destination = pairs.astype(np.int64, copy=False) + _, first = np.unique(source * width + destination, return_index=True) + else: + _, first = np.unique(pairs, axis=1, return_index=True) if first.size == pairs.shape[1]: return pairs return np.ascontiguousarray(pairs[:, np.sort(first)]) diff --git a/packages/experimental/kumo-pytorch/tests/test_diskgraph_graph_output.py b/packages/experimental/kumo-pytorch/tests/test_diskgraph_graph_output.py index 813b8cd57..d73a99daf 100644 --- a/packages/experimental/kumo-pytorch/tests/test_diskgraph_graph_output.py +++ b/packages/experimental/kumo-pytorch/tests/test_diskgraph_graph_output.py @@ -6,12 +6,13 @@ from types import SimpleNamespace +import numpy as np import pytest import torch from kumo.pytorch import api from kumo.pytorch.encoding import OutputBinding, fit_encoder -from kumo.pytorch.engines.diskgraph import DiskGraphSource +from kumo.pytorch.engines.diskgraph import DiskGraphSource, _first_occurrences pa = pytest.importorskip("pyarrow") @@ -188,3 +189,22 @@ def _identity_graph(column): ), } return source._graph_output("users", SimpleNamespace(layers=()), [event], {}, {}, {}) + + +@pytest.mark.parametrize( + "pairs", + [ + np.random.default_rng(0).integers(0, 40, (2, 2000)), + np.random.default_rng(1).integers(0, 2**20, (2, 50)).repeat(3, axis=1).astype(np.uint64), + np.array([[-1, 2, -1], [3, 4, 3]]), + np.array([[2**62, 0, 2**62], [1, 1, 1]]), + np.array([[1, 2**63, 1], [0, 0, 0]], dtype=np.uint64), + np.empty((2, 0), dtype=np.int64), + ], +) +def test_first_occurrences_keeps_first_unique_columns_in_order(pairs): + keys = [tuple(column) for column in pairs.T] + expected = [keys.index(key) for key in dict.fromkeys(keys)] + result = _first_occurrences(pairs) + assert result.dtype == pairs.dtype + np.testing.assert_array_equal(result, pairs[:, expected])