From 9eef14de0d0ab2430f903ba1e7b716f4b6838b9c Mon Sep 17 00:00:00 2001 From: Koussay-mehdouani Date: Mon, 9 Nov 2026 23:33:55 +0100 Subject: [PATCH 1/4] Honor random_seed in Snowflake entity sampling Snowflake supports SEED only for SYSTEM (block) sampling, not for a fixed row count, so a seeded draw now orders by a hash of the primary key and the seed and takes the first num_rows rows instead of the unseeded SAMPLE ROW clause. The unseeded path is unchanged. Fixes #89. Signed-off-by: Koussay-mehdouani --- .../rfm/backend/snow/sampler.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py index ed95c0265..0811ce983 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py @@ -84,10 +84,12 @@ def _sample_entity_table( random_seed: int | None = None, entity_ids: list | None = None, ) -> pd.DataFrame: - # NOTE Snowflake does support `SEED` only as part of `SYSTEM` sampling, - # which samples blocks rather than a fixed number of rows. - if entity_ids is None: - self._warn_random_seed_unsupported(random_seed) + # NOTE Snowflake supports `SEED` only as part of `SYSTEM` sampling, + # which samples blocks rather than a fixed number of rows. A seeded + # draw therefore orders by a hash of the primary key and the seed + # and takes the first `num_rows` rows instead: deterministic across + # calls, unlike `SAMPLE ROW`, which ignores the seed entirely. + seeded = entity_ids is None and random_seed is not None num_rows = min(num_rows, 1_000_000) # Snowflake's upper limit. @@ -120,7 +122,7 @@ def _sample_entity_table( self.table_column_proj_dict[table_name][column] for column in self._ordered_columns(table_name, columns) ] - if entity_ids is not None: + if entity_ids is not None or seeded: sql = ( f'SELECT {", ".join(projections)}\n' f'FROM {self.source_name_dict[table_name]}' @@ -133,6 +135,10 @@ def _sample_entity_table( ) if len(filters) > 0: sql += f'\nWHERE{" AND".join(filters)}' + if seeded: + key_ref = self.table_column_ref_dict[table_name][key] + sql += f'\nORDER BY HASH({key_ref}, {random_seed})' + sql += f'\nLIMIT {num_rows}' with paramstyle(self._connection), self._connection.cursor() as cursor: # NOTE This may return duplicate primary keys. This is okay. From 2a2ed3c92c2cfe4de296d0cb291283f2d654957d Mon Sep 17 00:00:00 2001 From: Koussay-mehdouani Date: Mon, 9 Nov 2026 23:34:13 +0100 Subject: [PATCH 2/4] Add regression test for seeded Snowflake sampling Asserts that a seeded draw does not warn, orders by HASH(primary key, seed) with a LIMIT instead of SAMPLE ROW, and that the seed is part of the ordering. Signed-off-by: Koussay-mehdouani --- .../tests/backend/test_sampler_entity_ids.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py index 2217d8a11..5820c6c37 100644 --- a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py +++ b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py @@ -8,6 +8,7 @@ from __future__ import annotations import uuid +import warnings from collections.abc import Iterator from pathlib import Path from typing import Any @@ -310,6 +311,37 @@ def test_snow_random_sample_still_binds_nothing(snow_sampler: Any) -> None: assert parameters is None +def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: + # Regression test: a seeded draw must not warn that the seed is ignored, + # and the SQL must pin the sample deterministically (a hash of the + # primary key and the seed) instead of the unseeded `SAMPLE ROW` clause. + snow_sampler._warned_random_seed = False + with warnings.catch_warnings(): + warnings.simplefilter('error') + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=10, + random_seed=42, + ) + + sql, parameters, _ = snow_sampler._connection.calls[-1] + assert 'SAMPLE ROW' not in sql + assert 'ORDER BY HASH("user_id", 42)' in sql + assert 'LIMIT 10' in sql + assert parameters is None + + # The seed is part of the ordering, so another seed reorders the draw: + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=10, + random_seed=7, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert 'ORDER BY HASH("user_id", 7)' in sql + + def test_snow_min_max_probe_keeps_table_names_out_of_the_sql( snow_sampler: Any, ) -> None: From d5bd3d51b1e7e82935ef4f01e6d0c63794720ce1 Mon Sep 17 00:00:00 2001 From: Koussay-mehdouani Date: Tue, 10 Nov 2026 08:28:01 +0100 Subject: [PATCH 3/4] Address CodeRabbit review: deterministic tie-break ordering and docstring for seeded Snowflake sampling Signed-off-by: Koussay-mehdouani Co-authored-by: Muse Spark (Meta AI) --- .../rfm/backend/snow/sampler.py | 19 ++++++++++++++++++- .../tests/backend/test_sampler_entity_ids.py | 4 ++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py index 0811ce983..051022060 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py @@ -84,6 +84,15 @@ def _sample_entity_table( random_seed: int | None = None, entity_ids: list | None = None, ) -> pd.DataFrame: + """Sample rows of an entity table from Snowflake. + + With ``entity_ids``, exactly those entities are fetched. With a + ``random_seed`` (and no ``entity_ids``), the draw is deterministic: + rows are ordered by a hash of the primary key and the seed, with + ties broken by the time column (when the table has one) and then + by the key itself, and the first ``num_rows`` rows are taken. + Without a seed, the draw uses Snowflake's unseeded ``SAMPLE ROW``. + """ # NOTE Snowflake supports `SEED` only as part of `SYSTEM` sampling, # which samples blocks rather than a fixed number of rows. A seeded # draw therefore orders by a hash of the primary key and the seed @@ -137,7 +146,15 @@ def _sample_entity_table( sql += f'\nWHERE{" AND".join(filters)}' if seeded: key_ref = self.table_column_ref_dict[table_name][key] - sql += f'\nORDER BY HASH({key_ref}, {random_seed})' + # Break hash ties deterministically (duplicate keys share a + # hash): by the time column when the table has one, then by + # the key itself, so a given seed always selects the same rows. + ordering = [f'HASH({key_ref}, {random_seed})'] + if column is not None: + ordering.append(self.table_column_ref_dict[table_name][column]) + ordering.append(key_ref) + order_by = ', '.join(ordering) + sql += f'\nORDER BY {order_by}' sql += f'\nLIMIT {num_rows}' with paramstyle(self._connection), self._connection.cursor() as cursor: diff --git a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py index 5820c6c37..79dc62464 100644 --- a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py +++ b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py @@ -327,7 +327,7 @@ def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: sql, parameters, _ = snow_sampler._connection.calls[-1] assert 'SAMPLE ROW' not in sql - assert 'ORDER BY HASH("user_id", 42)' in sql + assert 'ORDER BY HASH("user_id", 42), "ts", "user_id"' in sql assert 'LIMIT 10' in sql assert parameters is None @@ -339,7 +339,7 @@ def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: random_seed=7, ) sql, _, _ = snow_sampler._connection.calls[-1] - assert 'ORDER BY HASH("user_id", 7)' in sql + assert 'ORDER BY HASH("user_id", 7), "ts", "user_id"' in sql def test_snow_min_max_probe_keeps_table_names_out_of_the_sql( From bc746bec6e86b3372f9a3e6732047e20dc3d63a2 Mon Sep 17 00:00:00 2001 From: Koussay-mehdouani Date: Tue, 10 Nov 2026 09:14:02 +0100 Subject: [PATCH 4/4] Make the seeded Snowflake limit count distinct keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The caller drops duplicate keys after sampling, keeping the first row per key, so a seeded draw that limits rows before deduplication can return fewer distinct entities than requested (and silently weaken the under-coverage warnings gated on the deduplicated count). The seeded query now reduces to one representative row per key first — the earliest-time row, the one the caller would have kept, with the remaining columns as deterministic tie-breakers — via QUALIFY ROW_NUMBER(), then orders by HASH(key, seed) and only then applies LIMIT. The representative choice does not depend on the seed. Tests assert the deduplication clause precedes the ordering and the limit, and cover the multi-column tie-break. Signed-off-by: Koussay-mehdouani Co-authored-by: Muse Spark (Meta AI) --- .../rfm/backend/snow/sampler.py | 42 +++++++--- .../tests/backend/test_sampler_entity_ids.py | 83 ++++++++++++++++++- 2 files changed, 110 insertions(+), 15 deletions(-) diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py index 051022060..8305ce5dd 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/sampler.py @@ -87,11 +87,13 @@ def _sample_entity_table( """Sample rows of an entity table from Snowflake. With ``entity_ids``, exactly those entities are fetched. With a - ``random_seed`` (and no ``entity_ids``), the draw is deterministic: - rows are ordered by a hash of the primary key and the seed, with - ties broken by the time column (when the table has one) and then - by the key itself, and the first ``num_rows`` rows are taken. - Without a seed, the draw uses Snowflake's unseeded ``SAMPLE ROW``. + ``random_seed`` (and no ``entity_ids``), the draw is deterministic + and spends its limit on distinct entities: rows are first reduced + to one representative row per primary key (the earliest-time row, + chosen deterministically and independently of the seed), then + ordered by a hash of the key and the seed, and the first + ``num_rows`` are taken. Without a seed, the draw uses Snowflake's + unseeded ``SAMPLE ROW``. """ # NOTE Snowflake supports `SEED` only as part of `SYSTEM` sampling, # which samples blocks rather than a fixed number of rows. A seeded @@ -146,15 +148,29 @@ def _sample_entity_table( sql += f'\nWHERE{" AND".join(filters)}' if seeded: key_ref = self.table_column_ref_dict[table_name][key] - # Break hash ties deterministically (duplicate keys share a - # hash): by the time column when the table has one, then by - # the key itself, so a given seed always selects the same rows. - ordering = [f'HASH({key_ref}, {random_seed})'] + # The caller drops duplicate keys, keeping the first row per + # key, so the limit must count distinct keys rather than + # rows: first reduce to one representative row per key. The + # representative is the row the caller would have kept — + # earliest time first under the hash ordering — with the + # remaining columns as tie-breakers, so the choice is + # deterministic and does not depend on the seed. + window_order: list[str] = [] if column is not None: - ordering.append(self.table_column_ref_dict[table_name][column]) - ordering.append(key_ref) - order_by = ', '.join(ordering) - sql += f'\nORDER BY {order_by}' + window_order.append( + self.table_column_ref_dict[table_name][column] + ) + window_order.extend( + self.table_column_ref_dict[table_name][name] + for name in self._ordered_columns(table_name, columns) + if name != key and name != column + ) + partition_order = ', '.join(window_order) or key_ref + sql += ( + f'\nQUALIFY ROW_NUMBER() OVER (' + f'PARTITION BY {key_ref} ORDER BY {partition_order}) = 1' + ) + sql += f'\nORDER BY HASH({key_ref}, {random_seed})' sql += f'\nLIMIT {num_rows}' with paramstyle(self._connection), self._connection.cursor() as cursor: diff --git a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py index 79dc62464..c4850a344 100644 --- a/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py +++ b/packages/structured-data-models-relational-engine/tests/backend/test_sampler_entity_ids.py @@ -327,7 +327,7 @@ def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: sql, parameters, _ = snow_sampler._connection.calls[-1] assert 'SAMPLE ROW' not in sql - assert 'ORDER BY HASH("user_id", 42), "ts", "user_id"' in sql + assert 'ORDER BY HASH("user_id", 42)' in sql assert 'LIMIT 10' in sql assert parameters is None @@ -339,7 +339,86 @@ def test_snow_seeded_sample_orders_by_key_hash(snow_sampler: Any) -> None: random_seed=7, ) sql, _, _ = snow_sampler._connection.calls[-1] - assert 'ORDER BY HASH("user_id", 7), "ts", "user_id"' in sql + assert 'ORDER BY HASH("user_id", 7)' in sql + + +def test_snow_seeded_sample_limits_distinct_keys(snow_sampler: Any) -> None: + # The caller drops duplicate keys after sampling (keeping the first + # row per key), so a seeded draw must spend its LIMIT on distinct + # keys: one representative row per key is picked before the limit. + # The representative is the earliest-time row — the one the caller's + # drop_duplicates would keep — and the choice does not involve the + # seed, so it is stable across calls and across seeds. + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=5, + random_seed=42, + ) + sql, parameters, _ = snow_sampler._connection.calls[-1] + qualify = ( + 'QUALIFY ROW_NUMBER() OVER (PARTITION BY "user_id" ORDER BY "ts") = 1' + ) + assert qualify in sql + # Deduplication happens before ordering and limiting: + assert ( + sql.index('QUALIFY') < sql.index('ORDER BY HASH') < sql.index('LIMIT 5') + ) + assert parameters is None + + # Same seed-independence for another seed: identical QUALIFY clause, + # only the hash term changes: + snow_sampler._sample_entity_table( + table_name='users', + columns=_COLUMNS, + num_rows=5, + random_seed=7, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert qualify in sql + assert 'ORDER BY HASH("user_id", 7)' in sql + + +def test_snow_seeded_sample_tiebreaks_on_remaining_columns( + snow_sampler: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + # With a third column in the sample, the window ordering is the time + # column followed by the remaining columns in schema order, so rows + # sharing a key and a time still get a deterministic representative. + # The key and the time column appear exactly once each. + from sdm_relational_engine.rfm.base import SourceColumn + + snow_sampler._source_table_dict['users']['city'] = SourceColumn( + name='city', + dtype=Dtype.string, + is_primary_key=False, + is_unique_key=False, + is_nullable=False, + ) + snow_sampler._table_dtype_dict['users']['city'] = Dtype.string + snow_sampler._table_column_ref_dict['users']['city'] = '"city"' + + def fetch_with_city(self: Any, force_return_table: bool = True) -> pa.Table: + return pa.table( + { + 'user_id': pa.array([], type=pa.string()), + 'ts': pa.array([], type=pa.string()), + 'city': pa.array([], type=pa.string()), + } + ) + + monkeypatch.setattr(_SnowCursor, 'fetch_arrow_all', fetch_with_city) + snow_sampler._sample_entity_table( + table_name='users', + columns={'user_id', 'ts', 'city'}, + num_rows=5, + random_seed=42, + ) + sql, _, _ = snow_sampler._connection.calls[-1] + assert ( + 'QUALIFY ROW_NUMBER() OVER (PARTITION BY "user_id" ' + 'ORDER BY "ts", "city") = 1' + ) in sql def test_snow_min_max_probe_keeps_table_names_out_of_the_sql(