Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2561,7 +2561,12 @@ def _first_occurrences(pairs: np.ndarray) -> np.ndarray:

if pairs.shape[1] < 2:
return pairs
_, first = np.unique(pairs, axis=1, return_index=True)
lowest, source_max, width = int(pairs.min()), int(pairs[0].max()), int(pairs[1].max()) + 1
if lowest >= 0 and (source_max + 1) * width < 2**63:
source, destination = pairs.astype(np.int64, copy=False)
_, first = np.unique(source * width + destination, return_index=True)
else:
_, first = np.unique(pairs, axis=1, return_index=True)
if first.size == pairs.shape[1]:
return pairs
return np.ascontiguousarray(pairs[:, np.sort(first)])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@

from types import SimpleNamespace

import numpy as np
import pytest
import torch

from kumo.pytorch import api
from kumo.pytorch.encoding import OutputBinding, fit_encoder
from kumo.pytorch.engines.diskgraph import DiskGraphSource
from kumo.pytorch.engines.diskgraph import DiskGraphSource, _first_occurrences

pa = pytest.importorskip("pyarrow")

Expand Down Expand Up @@ -188,3 +189,22 @@ def _identity_graph(column):
),
}
return source._graph_output("users", SimpleNamespace(layers=()), [event], {}, {}, {})


@pytest.mark.parametrize(
"pairs",
[
np.random.default_rng(0).integers(0, 40, (2, 2000)),
np.random.default_rng(1).integers(0, 2**20, (2, 50)).repeat(3, axis=1).astype(np.uint64),
np.array([[-1, 2, -1], [3, 4, 3]]),
np.array([[2**62, 0, 2**62], [1, 1, 1]]),
np.array([[1, 2**63, 1], [0, 0, 0]], dtype=np.uint64),
np.empty((2, 0), dtype=np.int64),
],
)
def test_first_occurrences_keeps_first_unique_columns_in_order(pairs):
keys = [tuple(column) for column in pairs.T]
expected = [keys.index(key) for key in dict.fromkeys(keys)]
result = _first_occurrences(pairs)
assert result.dtype == pairs.dtype
np.testing.assert_array_equal(result, pairs[:, expected])
Loading