diff --git a/src/freshdata/cdc.py b/src/freshdata/cdc.py index 45964640..a2431122 100644 --- a/src/freshdata/cdc.py +++ b/src/freshdata/cdc.py @@ -27,7 +27,7 @@ import pandas as pd -from .streaming._timeseries import to_timedelta +from .streaming._timeseries import coerce_datetimes, to_timedelta __all__ = ["CDCDefect", "CDCReport", "cdc_profile"] @@ -156,6 +156,31 @@ def _sample_keys( return tuple(str(v) for v in vals) +def _parse_event_time(raw: pd.Series) -> pd.Series: + """Parse the event-time column; unparseable values become ``NaT``. + + Datetime columns pass through. Mixed UTC offsets (e.g. across a DST change) + or a mix of naive and offset-aware values are normalised to UTC, reading + naive values as UTC (see :func:`~freshdata.streaming._timeseries.coerce_datetimes`). + """ + return coerce_datetimes(raw) + + +def _to_event_tz(value: object, tz: Any) -> pd.Timestamp: + """Express a ``now`` / ``watermark`` reference in the event times' zone. + + Naive references and naive event times are both read as UTC, so a naive + value compared with tz-aware events is localised to UTC first, and an + aware value compared with naive events becomes naive UTC. + """ + stamp = pd.Timestamp(value) + if tz is None: + return stamp if stamp.tz is None else stamp.tz_convert("UTC").tz_localize(None) + if stamp.tz is None: + stamp = stamp.tz_localize("UTC") + return stamp.tz_convert(tz) + + def _running_max(ts: pd.Series, key_series: pd.Series | None) -> pd.Series: """Per-key (or global) running max of event time, in arrival order.""" if key_series is not None: @@ -214,7 +239,8 @@ def _freshness( max_ts = ts.max() if pd.isna(max_ts): return None - reference = now if now is not None else pd.Timestamp.now(tz=max_ts.tz) + # The default reference is the current UTC time (naive event times are UTC). + reference = now if now is not None else _to_event_tz(pd.Timestamp.now(tz="UTC"), max_ts.tz) freshness_seconds = float((reference - max_ts).total_seconds()) if stale_after_td is not None and freshness_seconds > stale_after_td.total_seconds(): defects.append( @@ -274,6 +300,8 @@ def cdc_profile( now, stale_after: Reference 'now' and the max age before the batch is ``stale``. ``now`` defaults to the current UTC time; pass it explicitly for determinism. + Naive ``now`` / ``watermark`` values and naive event times are read as + UTC; references are converted to the event times' zone before comparing. replay_threshold: Fraction of (duplicate + late) rows above which a ``replay_risk`` batch warning is raised. @@ -292,10 +320,10 @@ def cdc_profile( n_rows = len(df) lateness_td = to_timedelta(lateness) or pd.Timedelta(0) stale_after_td = to_timedelta(stale_after) - now_ts = pd.Timestamp(now) if now is not None else None - raw = df[event_time] - ts = raw if pd.api.types.is_datetime64_any_dtype(raw) else pd.to_datetime(raw, errors="coerce") + ts = _parse_event_time(df[event_time]) + event_tz = ts.dt.tz + now_ts = _to_event_tz(now, event_tz) if now is not None else None defects: list[CDCDefect] = [] @@ -315,7 +343,7 @@ def cdc_profile( # 2) ordering: out-of-order + late (relative to watermark / running watermark). if watermark is not None: - wm = pd.Timestamp(watermark) + wm = _to_event_tz(watermark, event_tz) late_mask = ts < wm n_late = int(late_mask.sum()) counts = {"out_of_order": 0, "late": n_late} diff --git a/src/freshdata/streaming/_timeseries.py b/src/freshdata/streaming/_timeseries.py index 8f2fd55b..05c8d10d 100644 --- a/src/freshdata/streaming/_timeseries.py +++ b/src/freshdata/streaming/_timeseries.py @@ -12,22 +12,25 @@ needs — a single per-stream watermark timestamp — so a :class:`StreamingCleaner` can run it batch-by-batch while still recognising events that arrive late relative to everything seen so far. Every transformation is audited as a :class:`~freshdata.report.Action` with -one of the ``timeseries_interpolation`` / ``seasonal_imputation`` / ``ordered_dedupe`` / -``late_data`` / ``windowed_anomaly`` step names, so the trust contract is preserved. +one of the ``timeseries_timestamp_parse`` / ``timeseries_interpolation`` / +``seasonal_imputation`` / ``ordered_dedupe`` / ``late_data`` / ``windowed_anomaly`` step +names, so the trust contract is preserved. """ from __future__ import annotations +import warnings from dataclasses import dataclass, field from typing import Any import numpy as np import pandas as pd -from .._util import safe_median +from .._util import mask_sensitive_value, safe_median from ..config import CleanConfig from ..engine.context import infer_role from ..report import CleanReport +from ..steps.dtypes import COERCED_CELLS_CAP #: Interpolation methods accepted by :class:`TimeSeriesCleanConfig`. INTERPOLATION_METHODS = ("linear", "time", "ffill", "bfill") @@ -39,6 +42,16 @@ ANOMALY_METHODS = ("rolling_zscore", "mad", "iqr", "ewma") #: What to do with a flagged anomaly. ANOMALY_ACTIONS = ("flag", "cap", "quarantine") +#: Epoch units accepted for numeric timestamp columns. +TIMESTAMP_UNITS = ("s", "ms", "us", "ns") +#: Upper bounds on the median |epoch value| for each inferred unit: seconds up to +#: 1e11 (year ~5100), milliseconds up to 1e14, microseconds up to 1e17, else ns. +_EPOCH_UNIT_BOUNDS = ((1e11, "s"), (1e14, "ms"), (1e17, "us")) +#: ``pd.to_datetime`` keyword sets :func:`coerce_datetimes` tries, in order. +#: ``format="ISO8601"`` exists (and is needed) only on pandas >= 2. +_DATETIME_PARSE_ATTEMPTS: tuple[dict[str, Any], ...] = ( + ({}, {"utc": True}, {"format": "ISO8601"}, {"format": "ISO8601", "utc": True}) + if int(pd.__version__.split(".")[0]) >= 2 else ({}, {"utc": True})) #: Named seasonal buckets → a function mapping a datetime index to a season key. _SEASON_KEYS = { @@ -61,6 +74,95 @@ def to_timedelta(value: object) -> pd.Timedelta | None: return pd.to_timedelta(value) +def _n_lost(values: pd.Series, parsed: pd.Series) -> int: + """How many present values the parse turned into ``NaT``.""" + return int((values.notna() & parsed.isna()).sum()) + + +def coerce_datetimes(values: pd.Series) -> pd.Series: + """Parse *values* with ``errors="coerce"``, surviving mixed UTC offsets. + + Datetime columns pass through unchanged. Values with different UTC offsets + (for example either side of a DST change), or a mix of naive and + offset-aware values, have no single naive or fixed-offset representation. + Depending on the pandas version, a plain parse then raises, returns an + ``object`` column, or turns the minority into ``NaT``. So when the plain + parse is unusable or loses values, the batch is re-parsed with ``utc=True`` + (reading naive values as UTC) and, on pandas >= 2, which infers one format + from the first value, also with ``format="ISO8601"``. The attempt that + loses the fewest values wins, the earliest on a tie, so clean batches pay + for one parse. Numeric input keeps pandas' default (nanoseconds); + :func:`parse_timestamps` infers epoch units. + """ + if pd.api.types.is_datetime64_any_dtype(values): + return values + best: pd.Series | None = None + best_lost = 0 + for kwargs in _DATETIME_PARSE_ATTEMPTS: + try: + with warnings.catch_warnings(): + # pandas 2 warns before returning an object column for mixed + # offsets; a later utc=True attempt handles exactly that case. + warnings.filterwarnings("ignore", message=".*mixed time zones.*") + parsed = pd.to_datetime(values, errors="coerce", **kwargs) + except (ValueError, TypeError): + continue + if not pd.api.types.is_datetime64_any_dtype(parsed): + continue # object column of mixed-offset datetimes + lost = _n_lost(values, parsed) + if best is None or lost < best_lost: + best, best_lost = parsed, lost + if not lost: + break + if best is None: # every attempt raised: surface the plain parse's error + return pd.to_datetime(values, errors="coerce") + return best + + +def infer_epoch_unit(values: pd.Series) -> str: + """Guess the epoch unit of numeric timestamps from their median magnitude.""" + present = values.dropna() + if not len(present): + return "ns" + magnitude = float(present.abs().median()) + for bound, unit in _EPOCH_UNIT_BOUNDS: + if magnitude < bound: + return unit + return "ns" + + +def parse_timestamps(values: pd.Series, unit: str | None = None + ) -> tuple[pd.Series, str | None]: + """Parse a timestamp column; return ``(parsed, epoch_unit)``. + + Numeric (non-bool) columns are epoch values in *unit*, or in the unit + :func:`infer_epoch_unit` infers when *unit* is ``None``; ``epoch_unit`` + reports the unit used. Other columns go through :func:`coerce_datetimes`, + and ``epoch_unit`` is ``None``. Values that do not parse (or are out of + range) become ``NaT``. + """ + dtype = values.dtype + if not (pd.api.types.is_numeric_dtype(dtype) and not pd.api.types.is_bool_dtype(dtype)): + return coerce_datetimes(values), None + unit = unit or infer_epoch_unit(values) + present = values.notna().to_numpy() + kept = values[present] + # Signed ints stay exact (ns epochs exceed float precision); everything else + # goes through float64 so nullable dtypes never hit pd.NA. + raw = (kept.to_numpy(dtype="int64") if pd.api.types.is_signed_integer_dtype(dtype) + else kept.to_numpy(dtype="float64")) + parsed = pd.Series(pd.NaT, index=values.index, dtype="datetime64[ns]") + if len(raw): + parsed[present] = pd.to_datetime(raw, unit=unit, errors="coerce").to_numpy() + return parsed, unit + + +def _as_float(s: pd.Series) -> pd.Series: + """Score a column as float64; non-numeric cells (e.g. a stray string) become NaN.""" + numeric = pd.to_numeric(s, errors="coerce") + return pd.Series(numeric.to_numpy(dtype="float64", na_value=np.nan), index=s.index) + + @dataclass(frozen=True) class TimeSeriesCleanConfig: """How a time-series / streaming-aware clean treats ordered, timestamped data. @@ -118,6 +220,11 @@ class TimeSeriesCleanConfig: Numeric column whose larger value wins ``highest_quality`` dedupe ties. protected_columns: Extra columns never interpolated, seasonally filled, or anomaly-scored. + timestamp_unit: + Epoch unit (one of :data:`TIMESTAMP_UNITS`) for *numeric* timestamp and + event-time columns. ``None`` (default) infers it from the magnitude of the + values (seconds, milliseconds, microseconds or nanoseconds). Ignored for + string and datetime columns. """ timestamp_column: str @@ -139,6 +246,7 @@ class TimeSeriesCleanConfig: anomaly_action: str = "flag" quality_column: str | None = None protected_columns: tuple[str, ...] = () + timestamp_unit: str | None = None def __post_init__(self) -> None: if not isinstance(self.timestamp_column, str) or not self.timestamp_column: @@ -150,8 +258,12 @@ def __post_init__(self) -> None: _check("anomaly_action", self.anomaly_action, ANOMALY_ACTIONS) if self.max_interpolation_gap < 0: raise ValueError("max_interpolation_gap must be >= 0") - if self.anomaly_window_size < 0: - raise ValueError("anomaly_window_size must be >= 0") + if self.anomaly_window_size < 0 or self.anomaly_window_size == 1: + # A one-row window has no spread to score against (and the rolling + # min_periods of 2 would exceed it), so 0 (disabled) or >= 2 only. + raise ValueError("anomaly_window_size must be 0 or >= 2") + if self.timestamp_unit is not None: + _check("timestamp_unit", self.timestamp_unit, TIMESTAMP_UNITS) if self.anomaly_threshold <= 0: raise ValueError("anomaly_threshold must be > 0") if (self.ordered_dedupe_keep == "highest_quality" @@ -226,7 +338,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *, # Nothing time-series-shaped about this batch; leave it untouched. self.last_summary = {} return df, _empty_like(df) - df[ts_col] = pd.to_datetime(df[ts_col], errors="coerce") + df[ts_col] = self._parse_timestamp_column(df[ts_col], report) # 1. Late data is judged in arrival order, before any sort reorders the batch. df, late_exc, late_meta = self._handle_late_data(df, report) @@ -244,6 +356,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *, roles = roles or self._infer_roles(df) numeric_cols = self.numeric_targets(df, roles) self.last_numeric_cols = [str(c) for c in numeric_cols] + anomaly_cols = self.anomaly_targets(df, roles) # 3. Short-gap interpolation, then 4. seasonal imputation of what's left. df = self._interpolate(df, numeric_cols, report) @@ -251,7 +364,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *, df = self._seasonal_impute(df, numeric_cols, report) # 5. Windowed anomaly detection (flag columns by default; cap/quarantine opt-in). - df, anom_exc, anom_meta = self._windowed_anomaly(df, numeric_cols, report) + df, anom_exc, anom_meta = self._windowed_anomaly(df, anomaly_cols, report) if anom_exc is not None and len(anom_exc): cleaned_exc.append(anom_exc) summary.update(anom_meta) @@ -263,6 +376,49 @@ def process(self, df: pd.DataFrame, report: CleanReport, *, self.last_summary = summary return df, exceptions + def _parse_timestamp_column(self, raw: pd.Series, report: CleanReport) -> pd.Series: + """Parse the timestamp column, auditing the epoch unit and every lost value. + + Rows whose timestamp does not parse are kept (with ``NaT``); their + original values are preserved in ``report.coerced_cells``. + """ + cfg = self.config + name = str(cfg.timestamp_column) + parsed, epoch_unit = parse_timestamps(raw, cfg.timestamp_unit) + if epoch_unit is not None: + source = ("timestamp_unit" if cfg.timestamp_unit + else "inferred from the magnitude of the values") + report.add("timeseries_timestamp_parse", + f"read numeric timestamps as epoch unit '{epoch_unit}'", + column=name, count=int(parsed.notna().sum()), risk="low", + rationale=f"epoch unit {source}") + lost = raw.notna() & parsed.isna() + n_lost = int(lost.sum()) + if not n_lost: + return parsed + originals = raw[lost] + sensitive = name in self.clean_config.sensitive_columns + cells = report.coerced_cells.setdefault(name, {}) + for row, value in originals.items(): + if len(cells) >= COERCED_CELLS_CAP: + break + cells[row] = mask_sensitive_value(value) if sensitive else value + report.coerced_rows[name] = tuple( + dict.fromkeys([*report.coerced_rows.get(name, ()), *originals.index])) + examples = ", ".join( + f"{mask_sensitive_value(v) if sensitive else repr(v)} (row {i})" + for i, v in list(originals.head(3).items())) + report.add("timeseries_timestamp_parse", + f"{n_lost} timestamp value(s) could not be parsed and were set " + "to missing; rows kept", column=name, count=n_lost, risk="medium", + rationale="unparseable or out-of-range timestamp; originals " + "preserved in report.coerced_cells") + report.add_warning( + f"column '{name}': {n_lost} timestamp value(s) could not be parsed " + f"and were set to missing, e.g. {examples}. Originals are preserved " + "in report.coerced_cells.") + return parsed + # -- step 5 (numbered by the spec): watermark-aware late data --------------- def _handle_late_data(self, df: pd.DataFrame, report: CleanReport @@ -274,7 +430,7 @@ def _handle_late_data(self, df: pd.DataFrame, report: CleanReport event_col = cfg.resolved_event_time_column if event_col not in df.columns: return df, None, {} - event_time = pd.to_datetime(df[event_col], errors="coerce") + event_time, _ = parse_timestamps(df[event_col], cfg.timestamp_unit) late_mask = self._late_mask(df, event_time, lateness) n_late = int(late_mask.sum()) @@ -512,7 +668,7 @@ def _windowed_anomaly(self, df: pd.DataFrame, numeric_cols: list[str], def score_group(g: pd.DataFrame, col: str = col, flags: pd.Series = flags, lower_all: pd.Series = lower_all, upper_all: pd.Series = upper_all) -> pd.DataFrame: - s = g[col].astype("float64") + s = _as_float(g[col]) f, lo, hi = _anomaly_scores(s, win, thr, method) flags.loc[g.index] = f.to_numpy() lower_all.loc[g.index] = lo.to_numpy() @@ -529,7 +685,7 @@ def score_group(g: pd.DataFrame, col: str = col, flags: pd.Series = flags, self.anomalies_flagged_total += n_flag action_note = "flagged" if cfg.anomaly_action == "cap": - capped = df[col].astype("float64").clip(lower=lower_all, upper=upper_all) + capped = _as_float(df[col]).clip(lower=lower_all, upper=upper_all) df.loc[flags, col] = capped[flags] report.outliers_handled += n_flag action_note = "flagged and capped" @@ -579,6 +735,26 @@ def numeric_targets(self, df: pd.DataFrame, impute, so the time-series policy (short-gap interpolation, long-gap preservation) owns their missing values. """ + return [c for c in self._numeric_role_columns(df, roles) + if pd.api.types.is_numeric_dtype(df[c])] + + def anomaly_targets(self, df: pd.DataFrame, + roles: dict[str, str] | None = None) -> list[str]: + """Columns that get a ``_anomaly`` flag column. + + Chosen by (locked) role rather than by this batch's dtype: a numeric + column whose batch holds a stray string arrives as ``object``, and must + still be scored (non-numeric cells score as missing) so every batch emits + the same flag columns. + """ + return [c for c in self._numeric_role_columns(df, roles) + if pd.api.types.is_numeric_dtype(df[c]) + or pd.api.types.is_object_dtype(df[c]) + or pd.api.types.is_string_dtype(df[c])] + + def _numeric_role_columns(self, df: pd.DataFrame, + roles: dict[str, str] | None) -> list[Any]: + """Unprotected columns whose role is ``numeric``, as the frame's own labels.""" roles = roles or self._infer_roles(df) cfg = self.config protected = { @@ -593,8 +769,6 @@ def numeric_targets(self, df: pd.DataFrame, continue if roles.get(name) != "numeric": continue - if not pd.api.types.is_numeric_dtype(df[c]): - continue cols.append(c) # the frame's own label (may be an int), not str(c) return cols @@ -614,8 +788,16 @@ def _anomaly_scores(s: pd.Series, win: int, thr: float, method: str med = s.rolling(win, min_periods=min_p).median() mad = (s - med).abs().rolling(win, min_periods=min_p).median() robust_z = 0.6745 * (s - med) / mad.replace(0, np.nan) - lower, upper = med - thr * mad / 0.6745, med + thr * mad / 0.6745 - flags = _flag(robust_z, thr, s, med, mad) + # A zero MAD only means more than half the window equals the median, + # which is routine for sparse or two-valued series. The window is flat + # only when every *other* point in it (the win - 1 before this one) is + # identical; a deviating point there is a spike. Otherwise a zero MAD + # gives no scale, so the point is not flagged and has no fence to cap to. + others = s.shift(1).rolling(win - 1, min_periods=1) + flat = (mad == 0) & (others.max() == others.min()) + fence = mad.where((mad > 0) | flat) + lower, upper = med - thr * fence / 0.6745, med + thr * fence / 0.6745 + flags = _flag(robust_z, thr, s, med, flat) return flags, lower, upper if method == "ewma": # Judge each point against the EWMA *forecast* from prior points (shifted by one). @@ -625,24 +807,25 @@ def _anomaly_scores(s: pd.Series, win: int, thr: float, method: str std = s.ewm(span=win, min_periods=min_p).std().shift(1) z = (s - mean) / std.replace(0, np.nan) lower, upper = mean - thr * std, mean + thr * std - flags = _flag(z, thr, s, mean, std) + flags = _flag(z, thr, s, mean, std == 0) return flags, lower, upper # rolling_zscore (default) mean = s.rolling(win, min_periods=min_p).mean() std = s.rolling(win, min_periods=min_p).std() z = (s - mean) / std.replace(0, np.nan) lower, upper = mean - thr * std, mean + thr * std - flags = _flag(z, thr, s, mean, std) + flags = _flag(z, thr, s, mean, std == 0) return flags, lower, upper def _flag(z: pd.Series, thr: float, s: pd.Series, center: pd.Series, - scale: pd.Series) -> pd.Series: + flat: pd.Series) -> pd.Series: """Flag points beyond *thr* standard scores, plus the degenerate case where the - rolling scale is exactly zero (a flat window) yet the point still deviates from the - centre — there the z-score is NaN, so it would otherwise slip through unflagged.""" + window is *flat* (the caller decides what that means for its scale) yet the point + still deviates from the centre — there the z-score is NaN, so it would otherwise + slip through unflagged.""" beyond = (z.abs() > thr).fillna(False) - flat_spike = (scale.fillna(np.nan) == 0) & ((s - center).abs() > 0) + flat_spike = flat.fillna(False) & ((s - center).abs() > 0) return (beyond | flat_spike.fillna(False)) diff --git a/tests/test_cdc_profile.py b/tests/test_cdc_profile.py index 670ea132..18846d4c 100644 --- a/tests/test_cdc_profile.py +++ b/tests/test_cdc_profile.py @@ -2,6 +2,8 @@ from __future__ import annotations +import time + import pandas as pd import pytest @@ -154,3 +156,56 @@ def test_does_not_mutate_input(): before = df.copy(deep=True) fd.cdc_profile(df, event_time="event_ts", key="entity_id", now=NOW) pd.testing.assert_frame_equal(df, before) + + +# -- time zones (#233 part 4) ---------------------------------------------------- + +AWARE = _events(ts=pd.to_datetime(["2024-01-01 00:10", "2024-01-01 00:00"]).tz_localize("UTC")) + + +def test_default_now_is_utc_for_naive_event_times(monkeypatch): + if hasattr(time, "tzset"): # simulate a host west of UTC + monkeypatch.setenv("TZ", "America/New_York") + time.tzset() + try: + event = pd.Timestamp.now(tz="UTC").tz_localize(None) - pd.Timedelta("3h") + rep = fd.cdc_profile(_events(ts=[event]), event_time="ts", stale_after="1h") + finally: + monkeypatch.undo() + if hasattr(time, "tzset"): + time.tzset() + assert rep.freshness_seconds == pytest.approx(3 * 3600, abs=60) + assert any(d.kind == "stale" for d in rep.defects) + + +def test_default_now_with_aware_event_times(): + event = pd.Timestamp.now(tz="UTC") - pd.Timedelta("2h") + rep = fd.cdc_profile(_events(ts=[event.tz_convert("Asia/Kolkata")]), event_time="ts") + assert rep.freshness_seconds == pytest.approx(2 * 3600, abs=60) + + +def test_naive_now_with_aware_event_times_is_read_as_utc(): + rep = fd.cdc_profile(AWARE, event_time="ts", now="2024-01-02") + assert rep.freshness_seconds == pytest.approx(24 * 3600 - 600) + + +def test_aware_now_with_naive_event_times_is_converted_to_utc(): + df = _events(ts=pd.to_datetime(["2024-01-01 10:00"])) + rep = fd.cdc_profile(df, event_time="ts", now="2024-01-01T12:00:00+01:00") + assert rep.freshness_seconds == pytest.approx(3600) + + +def test_naive_watermark_with_aware_event_times(): + rep = fd.cdc_profile(AWARE, event_time="ts", watermark="2024-01-01 00:05", now=NOW) + late = [d for d in rep.defects if d.kind == "late"] + assert late and late[0].n_rows == 1 + + +def test_mixed_utc_offsets_are_normalised(): + mixed = _events(ts=["2024-03-10T01:30:00-05:00", "2024-03-10T03:30:00-04:00", + "2024-03-10T02:00:00-05:00"], k=["a", "a", "a"]) + rep = fd.cdc_profile(mixed, event_time="ts", key="k", + now=pd.Timestamp("2024-03-11", tz="UTC")) + # 06:30Z, 07:30Z, then 07:00Z: only the last row is behind the running max. + assert {d.kind: d.n_rows for d in rep.defects} == {"late": 1} + assert rep.freshness_seconds == pytest.approx(16.5 * 3600) diff --git a/tests/test_streaming_timeseries.py b/tests/test_streaming_timeseries.py index caf7545d..23885a05 100644 --- a/tests/test_streaming_timeseries.py +++ b/tests/test_streaming_timeseries.py @@ -11,7 +11,8 @@ import freshdata as fd from freshdata import StreamingCleaner, TimeSeriesCleanConfig -from freshdata.streaming._timeseries import to_timedelta +from freshdata.report import CleanReport +from freshdata.streaming._timeseries import TimeSeriesProcessor, coerce_datetimes, to_timedelta # -- config validation --------------------------------------------------------- @@ -347,3 +348,171 @@ def test_integer_column_labels_in_timeseries_mode(): assert 7 in out.columns and 8 in out.columns assert out[8].tolist() == [1.0, 2.0, 3.0, 4.0] assert any(a.step == "timeseries_interpolation" and a.column == "8" for a in report) + + +# -- timestamp parsing (#227, #250) --------------------------------------------- + +def _ts_clean(df, **kw): + cfg = TimeSeriesCleanConfig(timestamp_column="ts", **kw) + return StreamingCleaner(time_series_config=cfg, verbose=False).clean_batch(df) + + +@pytest.mark.parametrize("scale,unit", [(1, "s"), (10**3, "ms"), (10**6, "us"), (10**9, "ns")]) +def test_integer_epoch_timestamps_infer_unit(scale, unit): + epoch = [(1704067200 + 3600 * i) * scale for i in range(4)] + df = pd.DataFrame({"ts": epoch, "v": [1.0, 2.0, 3.0, 4.0]}) + out, report = _ts_clean(df) + assert out["ts"].tolist() == list(pd.date_range("2024-01-01", periods=4, freq="h")) + parse = [a for a in report.actions if a.step == "timeseries_timestamp_parse"] + assert [a.column for a in parse] == ["ts"] + assert f"'{unit}'" in parse[0].description + assert not report.coerced_cells + + +def test_float_and_nullable_epoch_timestamps_keep_rows(): + for values in ([1704067200.0, np.nan, 1704074400.0], + pd.array([1704067200, None, 1704074400], dtype="Int64")): + df = pd.DataFrame({"ts": values, "v": [1.0, 2.0, 3.0]}) + out, report = _ts_clean(df) + assert len(out) == 3 + assert out["ts"].iloc[0] == pd.Timestamp("2024-01-01 00:00") + assert not report.coerced_cells # a missing epoch is not a lost value + + +def test_explicit_timestamp_unit_overrides_inference(): + df = pd.DataFrame({"ts": [1704067200, 1704067260], "v": [1.0, 2.0]}) + out, _ = _ts_clean(df, timestamp_unit="ms") + assert out["ts"].iloc[0] == pd.Timestamp(1704067200, unit="ms") + with pytest.raises(ValueError, match="timestamp_unit"): + TimeSeriesCleanConfig(timestamp_column="ts", timestamp_unit="minutes") + + +def test_unparseable_timestamps_are_reported_and_rows_kept(): + df = pd.DataFrame({"ts": ["2024-01-01", "2024-13-45", "2024-01-03"], + "v": [1.0, 2.0, 3.0]}) + out, report = _ts_clean(df) + assert len(out) == 3 + assert out["ts"].isna().sum() == 1 + assert report.coerced_cells == {"ts": {1: "2024-13-45"}} + assert report.coerced_rows["ts"] == (1,) + assert report.to_dict()["coerced_cells"] == {"ts": {"1": "2024-13-45"}} + lost = [a for a in report.actions + if a.step == "timeseries_timestamp_parse" and a.count == 1] + assert lost and lost[0].risk == "medium" + assert any("2024-13-45" in w for w in report.warnings) + + +def test_out_of_range_epoch_is_reported(): + df = pd.DataFrame({"ts": [1704067200, 99_999_999_999], "v": [1.0, 2.0]}) + out, report = _ts_clean(df, timestamp_unit="s") + assert out["ts"].isna().sum() == 1 + assert list(report.coerced_cells["ts"].values()) == [99_999_999_999] + + +def test_unparseable_sensitive_timestamp_is_masked_in_report(): + proc =TimeSeriesProcessor(TimeSeriesCleanConfig(timestamp_column="ts"), + clean_config=fd.CleanConfig(sensitive_columns=("ts",))) + report = CleanReport() + df = pd.DataFrame({"ts": ["2024-01-01", "secret-value"], "v": [1.0, 2.0]}) + proc.process(df, report) + (masked,) = report.coerced_cells["ts"].values() + assert masked.startswith("[SENSITIVE:") + assert not any("secret-value" in w for w in report.warnings) + + +def test_mixed_utc_offsets_across_dst_are_parsed_to_utc(): + df = pd.DataFrame({"ts": ["2024-03-10T01:00:00-05:00", "2024-03-10T01:30:00-05:00", + "2024-03-10T03:00:00-04:00", "2024-03-10T03:30:00-04:00"], + "v": [1.0, None, 3.0, 4.0]}) + out, report = _ts_clean(df) + assert str(out["ts"].dt.tz) == "UTC" + assert out["ts"].tolist() == list( + pd.date_range("2024-03-10 06:00", periods=4, freq="30min", tz="UTC")) + assert out["v"].tolist() == pytest.approx([1.0, 2.0, 3.0, 4.0]) + assert not report.coerced_cells + + +def test_naive_and_offset_aware_timestamps_mix_parses_to_utc(): + df = pd.DataFrame({"ts": ["2024-03-10T01:00:00", "2024-03-10T03:00:00-04:00"], + "v": [1.0, 2.0]}) + out, report = _ts_clean(df) + assert out["ts"].tolist() == [pd.Timestamp("2024-03-10 01:00", tz="UTC"), + pd.Timestamp("2024-03-10 07:00", tz="UTC")] + assert not report.coerced_cells + + +def test_coerce_datetimes_keeps_clean_batches_in_their_own_zone(): + naive = pd.Series(["2024-01-01 00:00", "2024-01-02 00:00", None]) + assert coerce_datetimes(naive).dt.tz is None + one_offset = coerce_datetimes(pd.Series(["2024-01-01T00:00:00+05:30"] * 2)) + assert one_offset.iloc[0] == pd.Timestamp("2023-12-31 18:30", tz="UTC") + assert str(one_offset.dt.tz) != "UTC" # a single fixed offset is kept + garbage = coerce_datetimes(pd.Series(["2024-01-01", "not a date"])) + assert garbage.dt.tz is None and garbage.isna().tolist() == [False, True] + + +def test_epoch_event_time_column_drives_late_data(): + df = pd.DataFrame({"ts": pd.date_range("2024", periods=3, freq="h"), + "ev": [1704067200, 1704070800, 1704067210], "v": [1.0, 2.0, 3.0]}) + out, report = _ts_clean(df, event_time_column="ev", allowed_lateness="1m") + assert len(out) == 2 + assert any(a.step == "late_data" and a.count == 1 for a in report.actions) + + +# -- anomaly config and MAD (#290, #291) ---------------------------------------- + +def test_anomaly_window_size_one_is_rejected(): + with pytest.raises(ValueError, match="0 or >= 2"): + TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=1) + df = pd.DataFrame({"ts": pd.date_range("2024", periods=5, freq="h"), + "v": [1.0, 2.0, 3.0, 4.0, 5.0]}) + out, _ = _ts_clean(df, anomaly_window_size=2) + assert "v_anomaly" in out.columns + + +def test_mad_does_not_flag_minority_values_of_two_valued_series(): + v = [0.0, 0, 0, 1, 0, 0, 1, 0, 0, 0] * 3 + df = pd.DataFrame({"ts": pd.date_range("2024", periods=30, freq="h"), "v": v}) + out, _ = _ts_clean(df, anomaly_window_size=10, anomaly_method="mad", + anomaly_action="cap") + assert int(out["v_anomaly"].sum()) == 0 + assert out["v"].sum() == 6 + + +def test_mad_still_flags_and_caps_spike_in_flat_window(): + x = np.full(40, 5.0) + x[25] = 999.0 + df = pd.DataFrame({"ts": pd.date_range("2024", periods=40, freq="min"), "x": x}) + out, _ = _ts_clean(df, anomaly_window_size=10, anomaly_method="mad", + anomaly_action="cap") + assert out["x_anomaly"].tolist() == [i == 25 for i in range(40)] + assert out["x"].iloc[25] == 5.0 + + +# -- anomaly columns when batch dtypes change (#248 part b) ---------------------- + +def test_anomaly_columns_stable_when_numeric_column_holds_a_string(): + cfg = TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=20, + anomaly_method="mad") + cleaner = StreamingCleaner(time_series_config=cfg, verbose=False) + b1 = pd.DataFrame({"ts": ["2024-01-01", "2024-01-02", "2024-01-03"], + "a": [0, 1, 2], "b": [0, 10, 20]}) + b2 = pd.DataFrame({"ts": ["2024-01-04", "2024-01-05", "2024-01-06"], + "a": ["oops", 4, 5], "b": [30, 40, 50]}) + out1, _ = cleaner.clean_batch(b1) + out2, _ = cleaner.clean_batch(b2) + assert list(out2.columns) == list(out1.columns) == ["ts", "a", "b", "a_anomaly", "b_anomaly"] + assert out2["a"].tolist() == ["oops", 4, 5] + + +def test_non_numeric_cell_is_neither_scored_nor_capped(): + cfg = TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=5, + anomaly_method="mad", anomaly_action="cap") + cleaner = StreamingCleaner(time_series_config=cfg, verbose=False) + cleaner.clean_batch(pd.DataFrame({"ts": pd.date_range("2024-01", periods=8, freq="h"), + "a": [5.0] * 8})) # locks 'a' as numeric + b2 = pd.DataFrame({"ts": pd.date_range("2024-02", periods=8, freq="h"), + "a": [5, 5, 5, 5, "oops", 5, 999, 5]}) + out, _ = cleaner.clean_batch(b2) + assert out["a_anomaly"].tolist() == [i == 6 for i in range(8)] + assert out["a"].tolist() == [5, 5, 5, 5, "oops", 5, 5.0, 5]