diff --git a/CHANGELOG.md b/CHANGELOG.md index 14af03209..cd972ee35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,10 @@ these packages. All five distributions share one version and one entry. - A request's description leaves out a wire table carrying an integer past the float range on every pandas release. pandas 3.1 had started to build a frame around one, where earlier releases refused it. +- A Snowflake `TIME` column keeps its values. It was read as a timestamp, which + turned every time of day into `NaT`; it is now read as ISO text + (`HH:MM:SS`, with the fraction of a second where there is one), nulls + staying null, and is no longer taken for a time column. ## 1.0.0 - 2026-10-15 diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/table.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/table.py index 89c5e698a..3064b07fa 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/table.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/snow/table.py @@ -349,7 +349,9 @@ def _to_dtype(dtype: str | None) -> Dtype | None: if dtype.startswith('DATE') or dtype.startswith('TIMESTAMP'): return Dtype.date if dtype.startswith('TIME'): - return Dtype.time + # A time of day has no date, so it is no timestamp. It is read as + # its ISO text, which keeps every value and its precision: + return Dtype.string if dtype.startswith('VECTOR'): try: # Parse element data type from 'VECTOR(dtype, dimension)': dtype = dtype.split(',')[0].split('(')[1].strip() diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/table.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/table.py index 2f9e1fbcf..1747cb867 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/table.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/table.py @@ -1165,10 +1165,47 @@ def _from_decimal(ser: pd.Series, dtype: Dtype | None) -> pd.Series: except (ValueError, TypeError, ArithmeticError): return ser + def _time_of_day_to_text(ser: pd.Series) -> pd.Series: + r"""A time-of-day column as ISO text: ``HH:MM:SS``, followed by the + fraction of a second where there is one, exact to the unit of the + column. Nulls stay null. + """ + arrow_type = ser.dtype.pyarrow_dtype + ticks = 10 ** {'s': 0, 'ms': 3, 'us': 6, 'ns': 9}[arrow_type.unit] + width = len(str(ticks)) - 1 + raw = ser.astype( + pd.ArrowDtype( + pa.int32() if arrow_type.bit_width == 32 else pa.int64() + ) + ) + + def _format(value: int) -> str: + seconds, fraction = divmod(int(value), ticks) + minutes, second = divmod(seconds, 60) + hour, minute = divmod(minutes, 60) + text = f'{hour:02d}:{minute:02d}:{second:02d}' + if fraction: + text += '.' + str(fraction).zfill(width).rstrip('0') + return text + + return pd.Series( + [None if pd.isna(v) else _format(v) for v in raw], + index=ser.index, + name=ser.name, + dtype=pd.ArrowDtype(pa.string()), + ) + for column_name in df.columns: dtype = (dtype_dict or {}).get(column_name) stype = (stype_dict or {}).get(column_name) + if ( + dtype == Dtype.string + and isinstance(df[column_name].dtype, pd.ArrowDtype) + and pa.types.is_time(df[column_name].dtype.pyarrow_dtype) + ): + df[column_name] = _time_of_day_to_text(df[column_name]) + if isinstance( df[column_name].dtype, pd.ArrowDtype ) and pa.types.is_decimal(df[column_name].dtype.pyarrow_dtype): diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/metadata_inference/timestamp.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/metadata_inference/timestamp.py index fd803956a..3ad6a3911 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/metadata_inference/timestamp.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/metadata_inference/timestamp.py @@ -9,6 +9,8 @@ from dateutil.parser import UnknownTimezoneWarning from predictive_query.schema.typing import Dtype, Stype +_TIME_OF_DAY = r'\d{1,2}:\d{2}(:\d{2}(\.\d+)?)?' + def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool: if not Stype.timestamp.supports_dtype(dtype): @@ -38,6 +40,11 @@ def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool: ser = ser.astype(str) # Avoid parsing numbers as unix timestamps. + # A time of day names no moment, though `pd.to_datetime` places it on + # today's date: + if ser.str.fullmatch(_TIME_OF_DAY).all(): + return False + with warnings.catch_warnings(): warnings.simplefilter('ignore', UnknownTimezoneWarning) warnings.filterwarnings('ignore', message='Could not infer format') diff --git a/packages/structured-data-models-relational-engine/tests/backend/snow/test_snow_graph.py b/packages/structured-data-models-relational-engine/tests/backend/snow/test_snow_graph.py index 6d911dca0..825cea40f 100644 --- a/packages/structured-data-models-relational-engine/tests/backend/snow/test_snow_graph.py +++ b/packages/structured-data-models-relational-engine/tests/backend/snow/test_snow_graph.py @@ -9,6 +9,7 @@ import pandas as pd import pytest +from predictive_query.schema.typing import Dtype, Stype from sdm_relational_engine.exceptions import GraphConstructionError from sdm_relational_engine.rfm import Graph from sdm_relational_engine.rfm.base import KeyCount, count_keys @@ -712,3 +713,62 @@ def test_semantic_view_refuses_an_edit_stamp_on_a_table_it_relates_to( choice = graph['DATACENTERS'].time_column_choice assert choice is not None and choice.fallback == 'LAST_LOGIN' assert graph['DATACENTERS'].time_column is None + + +@pytest.mark.parametrize( + ('arrow_type', 'expected'), + [ + (pa.time64('ns'), '08:30:15.123456789'), + (pa.time64('us'), '08:30:15.123456'), + (pa.time32('ms'), '08:30:15.123'), + (pa.time32('s'), '08:30:15'), + ], +) +def test_time_columns_are_kept_as_text_not_dropped( + monkeypatch: pytest.MonkeyPatch, arrow_type: 'pa.DataType', expected: str +) -> None: + r"""A `TIME` column was read as a timestamp, which turned every value + into `NaT`. It is read as ISO text, exact to the column's precision, and + a null stays null. + """ + from sdm_relational_engine.rfm import KumoRelational + + ticks = 10 ** {'s': 0, 'ms': 3, 'us': 6, 'ns': 9}[arrow_type.unit] + value = (8 * 3600 + 30 * 60 + 15) * ticks + int( + expected.partition('.')[2].ljust(9, '0')[: len(str(ticks)) - 1] or 0 + ) + opens = pd.Series( + [None if i % 3 == 0 else value for i in range(12)], + dtype=pd.ArrowDtype(arrow_type), + ) + monkeypatch.setitem(_DATA, 'RACKS', _DATA['RACKS'].assign(OPENS_AT=opens)) + monkeypatch.setitem( + _COLUMN_TYPES, + 'RACKS', + {**_COLUMN_TYPES['RACKS'], 'OPENS_AT': 'TIME(9)'}, + ) + + with warnings.catch_warnings(): + warnings.simplefilter('error', UserWarning) + warnings.filterwarnings( + 'ignore', message='.*does not support seeded random sampling' + ) + graph = Graph.from_snowflake( + connection=_FakeConnection(), + database=_DATABASE, + schema=_SCHEMA, + tables=[dict(name='RACKS', primary_key='RACK_ID')], + infer_metadata=True, + verbose=False, + ) + assert graph['RACKS']['OPENS_AT'].dtype == Dtype.string + assert graph['RACKS']['OPENS_AT'].stype != Stype.timestamp + + sampler = KumoRelational(graph, verbose=False)._sampler + df = sampler._sample_entity_table('RACKS', {'RACK_ID', 'OPENS_AT'}, 12) + + got = df.set_index('RACK_ID')['OPENS_AT'] + for i in range(12): + want = None if i % 3 == 0 else expected + have = got[f'rack_{i}'] + assert pd.isna(have) if want is None else have == want, (i, have) diff --git a/packages/structured-data-models-relational-engine/tests/metadata_inference/test_timestamp.py b/packages/structured-data-models-relational-engine/tests/metadata_inference/test_timestamp.py index 25c5818c7..775746c75 100644 --- a/packages/structured-data-models-relational-engine/tests/metadata_inference/test_timestamp.py +++ b/packages/structured-data-models-relational-engine/tests/metadata_inference/test_timestamp.py @@ -17,3 +17,10 @@ def test_contains_timestamp() -> None: assert not contains_timestamp(invalid_ser, 'datetime', dtype=Dtype.string) assert contains_timestamp(valid_ser, 'test', dtype=Dtype.string) assert not contains_timestamp(invalid_ser, 'test', dtype=Dtype.string) + + +def test_times_of_day_are_no_timestamp() -> None: + ser = pd.Series(['08:30:15', '23:59:59.123456', None, '00:00:00']) + + assert not contains_timestamp(ser, 'opens_at', dtype=Dtype.string) + assert not contains_timestamp(ser, 'time', dtype=Dtype.string)