Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,10 @@ these packages. All five distributions share one version and one entry.
- A request's description leaves out a wire table carrying an integer past
the float range on every pandas release. pandas 3.1 had started to build
a frame around one, where earlier releases refused it.
- A Snowflake `TIME` column keeps its values. It was read as a timestamp, which
turned every time of day into `NaT`; it is now read as ISO text
(`HH:MM:SS`, with the fraction of a second where there is one), nulls
staying null, and is no longer taken for a time column.

## 1.0.0 - 2026-10-15

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -349,7 +349,9 @@ def _to_dtype(dtype: str | None) -> Dtype | None:
if dtype.startswith('DATE') or dtype.startswith('TIMESTAMP'):
return Dtype.date
if dtype.startswith('TIME'):
return Dtype.time
# A time of day has no date, so it is no timestamp. It is read as
# its ISO text, which keeps every value and its precision:
return Dtype.string
if dtype.startswith('VECTOR'):
try: # Parse element data type from 'VECTOR(dtype, dimension)':
dtype = dtype.split(',')[0].split('(')[1].strip()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1165,10 +1165,47 @@ def _from_decimal(ser: pd.Series, dtype: Dtype | None) -> pd.Series:
except (ValueError, TypeError, ArithmeticError):
return ser

def _time_of_day_to_text(ser: pd.Series) -> pd.Series:
r"""A time-of-day column as ISO text: ``HH:MM:SS``, followed by the
fraction of a second where there is one, exact to the unit of the
column. Nulls stay null.
"""
arrow_type = ser.dtype.pyarrow_dtype
ticks = 10 ** {'s': 0, 'ms': 3, 'us': 6, 'ns': 9}[arrow_type.unit]
width = len(str(ticks)) - 1
raw = ser.astype(
pd.ArrowDtype(
pa.int32() if arrow_type.bit_width == 32 else pa.int64()
)
)

def _format(value: int) -> str:
seconds, fraction = divmod(int(value), ticks)
minutes, second = divmod(seconds, 60)
hour, minute = divmod(minutes, 60)
text = f'{hour:02d}:{minute:02d}:{second:02d}'
if fraction:
text += '.' + str(fraction).zfill(width).rstrip('0')
return text

return pd.Series(
[None if pd.isna(v) else _format(v) for v in raw],
index=ser.index,
name=ser.name,
dtype=pd.ArrowDtype(pa.string()),
)

for column_name in df.columns:
dtype = (dtype_dict or {}).get(column_name)
stype = (stype_dict or {}).get(column_name)

if (
dtype == Dtype.string
and isinstance(df[column_name].dtype, pd.ArrowDtype)
and pa.types.is_time(df[column_name].dtype.pyarrow_dtype)
):
df[column_name] = _time_of_day_to_text(df[column_name])

if isinstance(
df[column_name].dtype, pd.ArrowDtype
) and pa.types.is_decimal(df[column_name].dtype.pyarrow_dtype):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
from dateutil.parser import UnknownTimezoneWarning
from predictive_query.schema.typing import Dtype, Stype

_TIME_OF_DAY = r'\d{1,2}:\d{2}(:\d{2}(\.\d+)?)?'


def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool:
if not Stype.timestamp.supports_dtype(dtype):
Expand Down Expand Up @@ -38,6 +40,11 @@ def contains_timestamp(ser: pd.Series, column_name: str, dtype: Dtype) -> bool:

ser = ser.astype(str) # Avoid parsing numbers as unix timestamps.

# A time of day names no moment, though `pd.to_datetime` places it on
# today's date:
if ser.str.fullmatch(_TIME_OF_DAY).all():
return False

with warnings.catch_warnings():
warnings.simplefilter('ignore', UnknownTimezoneWarning)
warnings.filterwarnings('ignore', message='Could not infer format')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import pandas as pd
import pytest
from predictive_query.schema.typing import Dtype, Stype
from sdm_relational_engine.exceptions import GraphConstructionError
from sdm_relational_engine.rfm import Graph
from sdm_relational_engine.rfm.base import KeyCount, count_keys
Expand Down Expand Up @@ -712,3 +713,62 @@ def test_semantic_view_refuses_an_edit_stamp_on_a_table_it_relates_to(
choice = graph['DATACENTERS'].time_column_choice
assert choice is not None and choice.fallback == 'LAST_LOGIN'
assert graph['DATACENTERS'].time_column is None


@pytest.mark.parametrize(
('arrow_type', 'expected'),
[
(pa.time64('ns'), '08:30:15.123456789'),
(pa.time64('us'), '08:30:15.123456'),
(pa.time32('ms'), '08:30:15.123'),
(pa.time32('s'), '08:30:15'),
],
)
def test_time_columns_are_kept_as_text_not_dropped(
monkeypatch: pytest.MonkeyPatch, arrow_type: 'pa.DataType', expected: str
) -> None:
r"""A `TIME` column was read as a timestamp, which turned every value
into `NaT`. It is read as ISO text, exact to the column's precision, and
a null stays null.
"""
from sdm_relational_engine.rfm import KumoRelational

ticks = 10 ** {'s': 0, 'ms': 3, 'us': 6, 'ns': 9}[arrow_type.unit]
value = (8 * 3600 + 30 * 60 + 15) * ticks + int(
expected.partition('.')[2].ljust(9, '0')[: len(str(ticks)) - 1] or 0
)
opens = pd.Series(
[None if i % 3 == 0 else value for i in range(12)],
dtype=pd.ArrowDtype(arrow_type),
)
monkeypatch.setitem(_DATA, 'RACKS', _DATA['RACKS'].assign(OPENS_AT=opens))
monkeypatch.setitem(
_COLUMN_TYPES,
'RACKS',
{**_COLUMN_TYPES['RACKS'], 'OPENS_AT': 'TIME(9)'},
)

with warnings.catch_warnings():
warnings.simplefilter('error', UserWarning)
warnings.filterwarnings(
'ignore', message='.*does not support seeded random sampling'
)
graph = Graph.from_snowflake(
connection=_FakeConnection(),
database=_DATABASE,
schema=_SCHEMA,
tables=[dict(name='RACKS', primary_key='RACK_ID')],
infer_metadata=True,
verbose=False,
)
assert graph['RACKS']['OPENS_AT'].dtype == Dtype.string
assert graph['RACKS']['OPENS_AT'].stype != Stype.timestamp

sampler = KumoRelational(graph, verbose=False)._sampler
df = sampler._sample_entity_table('RACKS', {'RACK_ID', 'OPENS_AT'}, 12)

got = df.set_index('RACK_ID')['OPENS_AT']
for i in range(12):
want = None if i % 3 == 0 else expected
have = got[f'rack_{i}']
assert pd.isna(have) if want is None else have == want, (i, have)
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,10 @@ def test_contains_timestamp() -> None:
assert not contains_timestamp(invalid_ser, 'datetime', dtype=Dtype.string)
assert contains_timestamp(valid_ser, 'test', dtype=Dtype.string)
assert not contains_timestamp(invalid_ser, 'test', dtype=Dtype.string)


def test_times_of_day_are_no_timestamp() -> None:
ser = pd.Series(['08:30:15', '23:59:59.123456', None, '00:00:00'])

assert not contains_timestamp(ser, 'opens_at', dtype=Dtype.string)
assert not contains_timestamp(ser, 'time', dtype=Dtype.string)