diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py index ed95c0265..8305ce5dd 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py @@ -84,10 +84,23 @@ def _sample_entity_table( random_seed: int | None = None, entity_ids: list | None = None, ) -> pd.DataFrame: - # NOTE Snowflake does support `SEED` only as part of `SYSTEM` sampling, - # which samples blocks rather than a fixed number of rows. - if entity_ids is None: - self._warn_random_seed_unsupported(random_seed) + """Sample rows of an entity table from Snowflake. + + With ``entity_ids``, exactly those entities are fetched. With a + ``random_seed`` (and no ``entity_ids``), the draw is deterministic + and spends its limit on distinct entities: rows are first reduced + to one representative row per primary key (the earliest-time row, + chosen deterministically and independently of the seed), then + ordered by a hash of the key and the seed, and the first + ``num_rows`` are taken. Without a seed, the draw uses Snowflake's + unseeded ``SAMPLE ROW``. + """ + # NOTE Snowflake supports `SEED` only as part of `SYSTEM` sampling, + # which samples blocks rather than a fixed number of rows. A seeded + # draw therefore orders by a hash of the primary key and the seed + # and takes the first `num_rows` rows instead: deterministic across + # calls, unlike `SAMPLE ROW`, which ignores the seed entirely. + seeded = entity_ids is None and random_seed is not None num_rows = min(num_rows, 1_000_000) # Snowflake's upper limit. @@ -120,7 +133,7 @@ def _sample_entity_table( self.table_column_proj_dict[table_name][column] for column in self._ordered_columns(table_name, columns) ] - if entity_ids is not None: + if entity_ids is not None or seeded: sql = ( f'SELECT {", ".join(projections)}\n' f'FROM {self.source_name_dict[table_name]}' @@ -133,6 +146,32 @@ def _sample_entity_table( ) if len(filters) > 0: sql += f'\nWHERE{" AND".join(filters)}' + if seeded: + key_ref = self.table_column_ref_dict[table_name][key] + # The caller drops duplicate keys, keeping the first row per + # key, so the limit must count distinct keys rather than + # rows: first reduce to one representative row per key. The + # representative is the row the caller would have kept — + # earliest time first under the hash ordering — with the + # remaining columns as tie-breakers, so the choice is + # deterministic and does not depend on the seed. + window_order: list[str] = [] + if column is not None: + window_order.append( + self.table_column_ref_dict[table_name][column] + ) + window_order.extend( + self.table_column_ref_dict[table_name][name] + for name in self._ordered_columns(table_name, columns) + if name != key and name != column + ) + partition_order = ', '.join(window_order) or key_ref + sql += ( + f'\nQUALIFY ROW_NUMBER() OVER (' + f'PARTITION BY {key_ref} ORDER BY {partition_order}) = 1' + ) + sql += f'\nORDER BY HASH({key_ref}, {random_seed})' + sql += f'\nLIMIT {num_rows}' with paramstyle(self._connection), self._connection.cursor() as cursor: # NOTE This may return duplicate primary keys. This is okay. diff --git a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py index 2217d8a11..c4850a344 100644 --- a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py +++ b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py @@ -8,6 +8,7 @@ from __future__ import annotations import uuid +import warnings from collections.abc import Iterator from pathlib import Path from typing import Any @@ -310,6 +311,116 @@ def test_snow_random_sample_still_binds_nothing(snow_sampler: Any) -> None: assert parameters is None +def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: + # Regression test: a seeded draw must not warn that the seed is ignored, + # and the SQL must pin the sample deterministically (a hash of the + # primary key and the seed) instead of the unseeded `SAMPLE ROW` clause. + snow_sampler._warned_random_seed = False + with warnings.catch_warnings(): + warnings.simplefilter('error') + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=10, + random_seed=42, + ) + + sql, parameters, _ = snow_sampler._connection.calls[-1] + assert 'SAMPLE ROW' not in sql + assert 'ORDER BY HASH("user_id", 42)' in sql + assert 'LIMIT 10' in sql + assert parameters is None + + # The seed is part of the ordering, so another seed reorders the draw: + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=10, + random_seed=7, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert 'ORDER BY HASH("user_id", 7)' in sql + + +def test_snow_seeded_sample_limits_distinct_keys(snow_sampler: Any) -> None: + # The caller drops duplicate keys after sampling (keeping the first + # row per key), so a seeded draw must spend its LIMIT on distinct + # keys: one representative row per key is picked before the limit. + # The representative is the earliest-time row — the one the caller's + # drop_duplicates would keep — and the choice does not involve the + # seed, so it is stable across calls and across seeds. + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=5, + random_seed=42, + ) + sql, parameters, _ = snow_sampler._connection.calls[-1] + qualify = ( + 'QUALIFY ROW_NUMBER() OVER (PARTITION BY "user_id" ORDER BY "ts") = 1' + ) + assert qualify in sql + # Deduplication happens before ordering and limiting: + assert ( + sql.index('QUALIFY') < sql.index('ORDER BY HASH') < sql.index('LIMIT 5') + ) + assert parameters is None + + # Same seed-independence for another seed: identical QUALIFY clause, + # only the hash term changes: + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=5, + random_seed=7, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert qualify in sql + assert 'ORDER BY HASH("user_id", 7)' in sql + + +def test_snow_seeded_sample_tiebreaks_on_remaining_columns( + snow_sampler: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + # With a third column in the sample, the window ordering is the time + # column followed by the remaining columns in schema order, so rows + # sharing a key and a time still get a deterministic representative. + # The key and the time column appear exactly once each. + from sdm_relational_engine.rfm.base import SourceColumn + + snow_sampler._source_table_dict['users']['city'] = SourceColumn( + name='city', + dtype=Dtype.string, + is_primary_key=False, + is_unique_key=False, + is_nullable=False, + ) + snow_sampler._table_dtype_dict['users']['city'] = Dtype.string + snow_sampler._table_column_ref_dict['users']['city'] = '"city"' + + def fetch_with_city(self: Any, force_return_table: bool = True) -> pa.Table: + return pa.table( + { + 'user_id': pa.array([], type=pa.string()), + 'ts': pa.array([], type=pa.string()), + 'city': pa.array([], type=pa.string()), + } + ) + + monkeypatch.setattr(_SnowCursor, 'fetch_arrow_all', fetch_with_city) + snow_sampler._sample_entity_table( + table_name='users', + columns={'user_id', 'ts', 'city'}, + num_rows=5, + random_seed=42, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert ( + 'QUALIFY ROW_NUMBER() OVER (PARTITION BY "user_id" ' + 'ORDER BY "ts", "city") = 1' + ) in sql + + def test_snow_min_max_probe_keeps_table_names_out_of_the_sql( snow_sampler: Any, ) -> None: