From f183c77575c9ce78c7f517d64306d547f38a0197 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Wed, 11 Nov 2026 09:29:58 -0800 Subject: [PATCH 1/2] Gather over frames without the warehouse package A schema built from frames names no source, so the relational gather takes the branch that reads nothing from a warehouse. It still imported the data providers, and each of those imports sdm_connectors at module scope, so a gather over tables already in memory raised ModuleNotFoundError on a client installed without a driver extra. The package is in no base dependency, and every extra carrying it also carries a driver the route never uses. run.py already names the rule this breaks, keeping its own import of the package under TYPE_CHECKING because a core route over frames needs none of it, and test_core_route_without_connectors.py already asserted it. That test only ran the tabular route, whose materializer builds the request directly and never reaches the gather, so the one route that could break it was the one route it did not cover. The providers module now imports a provider on the first read of its name, through the module's own attribute lookup so a test that binds one still wins, and the test runs both routes. Co-Authored-By: Claude Opus 5 Signed-off-by: Manush Murali --- .../sdm_client/_rust/providers/__init__.py | 38 ++++-- .../test_core_route_without_connectors.py | 111 +++++++++++++----- 2 files changed, 107 insertions(+), 42 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py b/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py index 4b5c6e3a2..20a345a49 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py @@ -15,17 +15,10 @@ from __future__ import annotations +import importlib +import sys from typing import Any -from sdm_client._rust.providers.databricks import ( - DatabricksSparkProvider, - DatabricksSqlProvider, -) -from sdm_client._rust.providers.duckdb import DuckDBProvider -from sdm_client._rust.providers.postgres import PostgresProvider -from sdm_client._rust.providers.snowflake import SnowflakeProvider -from sdm_client._rust.providers.sqlite import SqliteProvider - __all__ = [ 'DatabricksSparkProvider', 'DatabricksSqlProvider', @@ -49,6 +42,31 @@ 'snowflake': ('snowflake', 'SnowflakeProvider'), } +#: The module each provider class is written in. Every one of them imports the +#: warehouse package, which is an optional dependency, so a module is imported +#: when a run first names its kind of source rather than when this one is. A +#: route over frames names none of them and needs none of the package. +_WRITTEN_IN: dict[str, str] = { + 'DatabricksSparkProvider': 'databricks', + 'DatabricksSqlProvider': 'databricks', + 'DuckDBProvider': 'duckdb', + 'PostgresProvider': 'postgres', + 'SnowflakeProvider': 'snowflake', + 'SqliteProvider': 'sqlite', +} + + +def __getattr__(name: str) -> Any: + r"""A provider class, imported and kept on the first read of its name.""" + if name not in _WRITTEN_IN: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + module = importlib.import_module( + f'sdm_client._rust.providers.{_WRITTEN_IN[name]}' + ) + made = getattr(module, name) + globals()[name] = made + return made + def for_source(source: Any) -> tuple[dict[str, Any], Any]: r"""The provider choice a run over ``source`` names, and the provider @@ -87,7 +105,7 @@ def for_source(source: Any) -> tuple[dict[str, Any], Any]: 'has no provider for' ) dialect, name = _BY_KIND[kept.kind] - made = globals()[name] + made = getattr(sys.modules[__name__], name) choice['dialect'] = dialect parts = kept.namespace_parts() if any(part is not None for part in parts): diff --git a/packages/structured-data-models-client/tests/test_core_route_without_connectors.py b/packages/structured-data-models-client/tests/test_core_route_without_connectors.py index 475dbe1a4..c131bc7ac 100644 --- a/packages/structured-data-models-client/tests/test_core_route_without_connectors.py +++ b/packages/structured-data-models-client/tests/test_core_route_without_connectors.py @@ -4,7 +4,11 @@ r"""A core route over frames runs where the warehouse package (``sdm_connectors``, an optional dependency of the client) is not installed -: the run loop imports nothing of it.""" +: the run loop imports nothing of it. + +Both models are held to it. The relational route gathers through +``_rust.frame_materialize``, which the tabular route does not reach, so each +needs its own run to say the package stayed out of it.""" from __future__ import annotations @@ -12,39 +16,82 @@ import sys import textwrap +import pytest -def test_an_in_memory_core_query_runs_without_the_connectors_package() -> None: - script = textwrap.dedent( - """ - import importlib.abc - import sys - - class Absent(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname.split('.')[0] == 'sdm_connectors': - raise ModuleNotFoundError( - "No module named 'sdm_connectors'", name=fullname - ) - - for name in [m for m in sys.modules if m.split('.')[0] == 'sdm_connectors']: - del sys.modules[name] - sys.meta_path.insert(0, Absent()) - - import pandas as pd - from sdm_client import KumoTabular, PredictiveQuery, Schema - - schema = Schema.from_data( - {'users': pd.DataFrame({'id': [1, 2, 3, 4], 'age': [10.5, 20.5, 30.5, 40.5]})} - ) - request = PredictiveQuery( - 'PREDICT users.age FOR EACH users.id', schema - ).materialize(model=KumoTabular(), indices=[1]) - assert 'sdm_connectors' not in sys.modules - print(type(request).__name__) - """ - ) +ABSENT = """ +import importlib.abc +import sys + +class Absent(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] == 'sdm_connectors': + raise ModuleNotFoundError( + "No module named 'sdm_connectors'", name=fullname + ) + +for name in [m for m in sys.modules if m.split('.')[0] == 'sdm_connectors']: + del sys.modules[name] +sys.meta_path.insert(0, Absent()) + +import pandas as pd +""" + +TABULAR = """ +from sdm_client import KumoTabular, PredictiveQuery, Schema + +schema = Schema.from_data( + {'users': pd.DataFrame({'id': [1, 2, 3, 4], 'age': [10.5, 20.5, 30.5, 40.5]})} +) +request = PredictiveQuery( + 'PREDICT users.age FOR EACH users.id', schema +).materialize(model=KumoTabular(), indices=[1]) +""" + +# The relational gather reads the client's data providers to find the one the +# schema's source names. A schema built from frames names no source, so it +# must reach none of them, and none of the warehouse package they import. +RELATIONAL = """ +from sdm_client import ( + KumoRelational, + PredictiveQuery, + Relationship, + Schema, +) + +users = pd.DataFrame({'user_id': range(12)}) +orders = pd.DataFrame( + { + 'order_id': range(120), + 'user_id': [i % 12 for i in range(120)], + 'ordered_at': pd.date_range('2026-01-01', periods=120, freq='D'), + } +) +schema = Schema.from_data( + {'users': users, 'orders': orders}, + primary_keys={'users': 'user_id', 'orders': 'order_id'}, + time_columns={'orders': 'ordered_at'}, + relationships=[Relationship('orders', 'user_id', 'users', 'user_id')], +) +assert schema.source is None +request = PredictiveQuery( + 'PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', schema +).materialize(model=KumoRelational(), indices=[1, 2, 3]) +""" + +REPORT = """ +assert 'sdm_connectors' not in sys.modules +print(type(request).__name__) +""" + + +@pytest.mark.parametrize( + 'route', [TABULAR, RELATIONAL], ids=['tabular', 'relational'] +) +def test_an_in_memory_core_query_runs_without_the_connectors_package( + route: str, +) -> None: result = subprocess.run( - [sys.executable, '-c', script], + [sys.executable, '-c', textwrap.dedent(ABSENT + route + REPORT)], capture_output=True, text=True, timeout=300, From 48deae3e923db18b877d93f5247b0511c652a041 Mon Sep 17 00:00:00 2001 From: Manush Murali Date: Wed, 11 Nov 2026 20:38:42 -0800 Subject: [PATCH 2/2] Read a provider class afresh rather than binding it The lazy read bound the class it imported into this module's globals. A test that swaps a driver out and back imports the provider module again inside that window, so the binding outlived the module it came from and a later run answered with a class the swap had made. Python's module cache makes the repeated import a dictionary read. Signed-off-by: Manush Murali Co-Authored-By: Claude Opus 5 --- .../src/sdm_client/_rust/providers/__init__.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py b/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py index 20a345a49..b3d299a7b 100644 --- a/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py +++ b/packages/structured-data-models-client/src/sdm_client/_rust/providers/__init__.py @@ -57,15 +57,19 @@ def __getattr__(name: str) -> Any: - r"""A provider class, imported and kept on the first read of its name.""" + r"""A provider class, imported on each read of its name. + + The class read is not bound here. A name bound on the first read would + outlive the module it was read from, and a run that follows a test which + swapped a driver out and back would answer with the class the swap made + rather than the one the module holds. + """ if name not in _WRITTEN_IN: raise AttributeError(f'module {__name__!r} has no attribute {name!r}') module = importlib.import_module( f'sdm_client._rust.providers.{_WRITTEN_IN[name]}' ) - made = getattr(module, name) - globals()[name] = made - return made + return getattr(module, name) def for_source(source: Any) -> tuple[dict[str, Any], Any]: