Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,10 @@

from __future__ import annotations

import importlib
import sys
from typing import Any

from sdm_client._rust.providers.databricks import (
DatabricksSparkProvider,
DatabricksSqlProvider,
)
from sdm_client._rust.providers.duckdb import DuckDBProvider
from sdm_client._rust.providers.postgres import PostgresProvider
from sdm_client._rust.providers.snowflake import SnowflakeProvider
from sdm_client._rust.providers.sqlite import SqliteProvider

__all__ = [
'DatabricksSparkProvider',
'DatabricksSqlProvider',
Expand All @@ -49,6 +42,35 @@
'snowflake': ('snowflake', 'SnowflakeProvider'),
}

#: The module each provider class is written in. Every one of them imports the
#: warehouse package, which is an optional dependency, so a module is imported
#: when a run first names its kind of source rather than when this one is. A
#: route over frames names none of them and needs none of the package.
_WRITTEN_IN: dict[str, str] = {
'DatabricksSparkProvider': 'databricks',
'DatabricksSqlProvider': 'databricks',
'DuckDBProvider': 'duckdb',
'PostgresProvider': 'postgres',
'SnowflakeProvider': 'snowflake',
'SqliteProvider': 'sqlite',
}


def __getattr__(name: str) -> Any:
r"""A provider class, imported on each read of its name.

The class read is not bound here. A name bound on the first read would
outlive the module it was read from, and a run that follows a test which
swapped a driver out and back would answer with the class the swap made
rather than the one the module holds.
"""
if name not in _WRITTEN_IN:
raise AttributeError(f'module {__name__!r} has no attribute {name!r}')
module = importlib.import_module(
f'sdm_client._rust.providers.{_WRITTEN_IN[name]}'
)
return getattr(module, name)


def for_source(source: Any) -> tuple[dict[str, Any], Any]:
r"""The provider choice a run over ``source`` names, and the provider
Expand Down Expand Up @@ -87,7 +109,7 @@ def for_source(source: Any) -> tuple[dict[str, Any], Any]:
'has no provider for'
)
dialect, name = _BY_KIND[kept.kind]
made = globals()[name]
made = getattr(sys.modules[__name__], name)
choice['dialect'] = dialect
parts = kept.namespace_parts()
if any(part is not None for part in parts):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,94 @@

r"""A core route over frames runs where the warehouse package
(``sdm_connectors``, an optional dependency of the client) is not installed
: the run loop imports nothing of it."""
: the run loop imports nothing of it.

Both models are held to it. The relational route gathers through
``_rust.frame_materialize``, which the tabular route does not reach, so each
needs its own run to say the package stayed out of it."""

from __future__ import annotations

import subprocess
import sys
import textwrap

import pytest

def test_an_in_memory_core_query_runs_without_the_connectors_package() -> None:
script = textwrap.dedent(
"""
import importlib.abc
import sys

class Absent(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname.split('.')[0] == 'sdm_connectors':
raise ModuleNotFoundError(
"No module named 'sdm_connectors'", name=fullname
)

for name in [m for m in sys.modules if m.split('.')[0] == 'sdm_connectors']:
del sys.modules[name]
sys.meta_path.insert(0, Absent())

import pandas as pd
from sdm_client import KumoTabular, PredictiveQuery, Schema

schema = Schema.from_data(
{'users': pd.DataFrame({'id': [1, 2, 3, 4], 'age': [10.5, 20.5, 30.5, 40.5]})}
)
request = PredictiveQuery(
'PREDICT users.age FOR EACH users.id', schema
).materialize(model=KumoTabular(), indices=[1])
assert 'sdm_connectors' not in sys.modules
print(type(request).__name__)
"""
)
ABSENT = """
import importlib.abc
import sys

class Absent(importlib.abc.MetaPathFinder):
def find_spec(self, fullname, path=None, target=None):
if fullname.split('.')[0] == 'sdm_connectors':
raise ModuleNotFoundError(
"No module named 'sdm_connectors'", name=fullname
)

for name in [m for m in sys.modules if m.split('.')[0] == 'sdm_connectors']:
del sys.modules[name]
sys.meta_path.insert(0, Absent())

import pandas as pd
"""

TABULAR = """
from sdm_client import KumoTabular, PredictiveQuery, Schema

schema = Schema.from_data(
{'users': pd.DataFrame({'id': [1, 2, 3, 4], 'age': [10.5, 20.5, 30.5, 40.5]})}
)
request = PredictiveQuery(
'PREDICT users.age FOR EACH users.id', schema
).materialize(model=KumoTabular(), indices=[1])
"""

# The relational gather reads the client's data providers to find the one the
# schema's source names. A schema built from frames names no source, so it
# must reach none of them, and none of the warehouse package they import.
RELATIONAL = """
from sdm_client import (
KumoRelational,
PredictiveQuery,
Relationship,
Schema,
)

users = pd.DataFrame({'user_id': range(12)})
orders = pd.DataFrame(
{
'order_id': range(120),
'user_id': [i % 12 for i in range(120)],
'ordered_at': pd.date_range('2026-01-01', periods=120, freq='D'),
}
)
schema = Schema.from_data(
{'users': users, 'orders': orders},
primary_keys={'users': 'user_id', 'orders': 'order_id'},
time_columns={'orders': 'ordered_at'},
relationships=[Relationship('orders', 'user_id', 'users', 'user_id')],
)
assert schema.source is None
request = PredictiveQuery(
'PREDICT COUNT(orders.*, 0, 30, days) FOR EACH users.user_id', schema
).materialize(model=KumoRelational(), indices=[1, 2, 3])
"""

REPORT = """
assert 'sdm_connectors' not in sys.modules
print(type(request).__name__)
"""


@pytest.mark.parametrize(
'route', [TABULAR, RELATIONAL], ids=['tabular', 'relational']
)
def test_an_in_memory_core_query_runs_without_the_connectors_package(
route: str,
) -> None:
result = subprocess.run(
[sys.executable, '-c', script],
[sys.executable, '-c', textwrap.dedent(ABSENT + route + REPORT)],
capture_output=True,
text=True,
timeout=300,
Expand Down
Loading