From 0ac8bc98a7f90d9e4a1a0075cc7021e3a53158ef Mon Sep 17 00:00:00 2001
From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com>
Date: Tue, 15 Sep 2026 03:03:43 +0530
Subject: [PATCH] fix(timeseries): parse event times faithfully and stop
spurious MAD flags
Time-series streaming parsed the timestamp column with a bare
pd.to_datetime(errors="coerce"). Integer epoch seconds were read as
nanoseconds (1970), unparseable strings silently became NaT, and a batch
with mixed UTC offsets (a DST change) raised ValueError.
- #227: numeric timestamp and event-time columns are read as epochs. The
unit comes from the new TimeSeriesCleanConfig.timestamp_unit
("s"/"ms"/"us"/"ns"), or is inferred from the median magnitude
(<1e11 s, <1e14 ms, <1e17 us, else ns); a "timeseries_timestamp_parse"
action records the unit. Values that still fail to parse keep their row
and are recorded in report.coerced_cells / coerced_rows (masked for
sensitive_columns) with a medium-risk action and a report warning.
- #250: string parsing goes through coerce_datetimes(). When the plain
parse raises, returns an object column, or loses values, it retries with
utc=True and, on pandas >= 2 (which infers one format from the first
value), with format="ISO8601", keeping the attempt that loses the fewest
values. Clean batches still pay for a single parse, and a single-offset
or naive batch keeps its zone.
- #290: anomaly_window_size=1 is rejected in the config ("must be 0 or
>= 2") instead of failing inside pandas on the first batch.
- #291: MAD treated a zero MAD as a flat window, so the minority value of
any sparse or two-valued series was flagged and, with cap, overwritten
by the median. A window is now flat only when every other point in it
is identical. A zero MAD in a non-flat window gives no scale, so the
point is not flagged and its fences are NaN. A spike in a truly flat
window is still flagged and capped.
- #248 part b: anomaly target columns are chosen from the (locked)
numeric role, not from the batch dtype, so a numeric column holding a
stray string still emits
_anomaly. It is scored with
pd.to_numeric(errors="coerce"): non-numeric cells are neither flagged
nor capped. Interpolation and seasonal imputation still require a
numeric dtype.
- #233 part 4: cdc_profile parses event times with the same
coerce_datetimes() helper, so mixed offsets no longer crash cummax. The
default now is the current UTC time instead of local wall-clock time
for naive events. now= and watermark= are converted to the event
times' zone, reading naive values (and naive events) as UTC.
Closes #227
Closes #250
Closes #290
Closes #291
Refs #248 (part b: time-series anomaly columns)
Refs #233 (part 4: cdc_profile)
---
src/freshdata/cdc.py | 40 ++++-
src/freshdata/streaming/_timeseries.py | 223 ++++++++++++++++++++++---
tests/test_cdc_profile.py | 55 ++++++
tests/test_streaming_timeseries.py | 171 ++++++++++++++++++-
4 files changed, 462 insertions(+), 27 deletions(-)
diff --git a/src/freshdata/cdc.py b/src/freshdata/cdc.py
index 45964640..a2431122 100644
--- a/src/freshdata/cdc.py
+++ b/src/freshdata/cdc.py
@@ -27,7 +27,7 @@
import pandas as pd
-from .streaming._timeseries import to_timedelta
+from .streaming._timeseries import coerce_datetimes, to_timedelta
__all__ = ["CDCDefect", "CDCReport", "cdc_profile"]
@@ -156,6 +156,31 @@ def _sample_keys(
return tuple(str(v) for v in vals)
+def _parse_event_time(raw: pd.Series) -> pd.Series:
+ """Parse the event-time column; unparseable values become ``NaT``.
+
+ Datetime columns pass through. Mixed UTC offsets (e.g. across a DST change)
+ or a mix of naive and offset-aware values are normalised to UTC, reading
+ naive values as UTC (see :func:`~freshdata.streaming._timeseries.coerce_datetimes`).
+ """
+ return coerce_datetimes(raw)
+
+
+def _to_event_tz(value: object, tz: Any) -> pd.Timestamp:
+ """Express a ``now`` / ``watermark`` reference in the event times' zone.
+
+ Naive references and naive event times are both read as UTC, so a naive
+ value compared with tz-aware events is localised to UTC first, and an
+ aware value compared with naive events becomes naive UTC.
+ """
+ stamp = pd.Timestamp(value)
+ if tz is None:
+ return stamp if stamp.tz is None else stamp.tz_convert("UTC").tz_localize(None)
+ if stamp.tz is None:
+ stamp = stamp.tz_localize("UTC")
+ return stamp.tz_convert(tz)
+
+
def _running_max(ts: pd.Series, key_series: pd.Series | None) -> pd.Series:
"""Per-key (or global) running max of event time, in arrival order."""
if key_series is not None:
@@ -214,7 +239,8 @@ def _freshness(
max_ts = ts.max()
if pd.isna(max_ts):
return None
- reference = now if now is not None else pd.Timestamp.now(tz=max_ts.tz)
+ # The default reference is the current UTC time (naive event times are UTC).
+ reference = now if now is not None else _to_event_tz(pd.Timestamp.now(tz="UTC"), max_ts.tz)
freshness_seconds = float((reference - max_ts).total_seconds())
if stale_after_td is not None and freshness_seconds > stale_after_td.total_seconds():
defects.append(
@@ -274,6 +300,8 @@ def cdc_profile(
now, stale_after:
Reference 'now' and the max age before the batch is ``stale``. ``now``
defaults to the current UTC time; pass it explicitly for determinism.
+ Naive ``now`` / ``watermark`` values and naive event times are read as
+ UTC; references are converted to the event times' zone before comparing.
replay_threshold:
Fraction of (duplicate + late) rows above which a ``replay_risk`` batch
warning is raised.
@@ -292,10 +320,10 @@ def cdc_profile(
n_rows = len(df)
lateness_td = to_timedelta(lateness) or pd.Timedelta(0)
stale_after_td = to_timedelta(stale_after)
- now_ts = pd.Timestamp(now) if now is not None else None
- raw = df[event_time]
- ts = raw if pd.api.types.is_datetime64_any_dtype(raw) else pd.to_datetime(raw, errors="coerce")
+ ts = _parse_event_time(df[event_time])
+ event_tz = ts.dt.tz
+ now_ts = _to_event_tz(now, event_tz) if now is not None else None
defects: list[CDCDefect] = []
@@ -315,7 +343,7 @@ def cdc_profile(
# 2) ordering: out-of-order + late (relative to watermark / running watermark).
if watermark is not None:
- wm = pd.Timestamp(watermark)
+ wm = _to_event_tz(watermark, event_tz)
late_mask = ts < wm
n_late = int(late_mask.sum())
counts = {"out_of_order": 0, "late": n_late}
diff --git a/src/freshdata/streaming/_timeseries.py b/src/freshdata/streaming/_timeseries.py
index 8f2fd55b..05c8d10d 100644
--- a/src/freshdata/streaming/_timeseries.py
+++ b/src/freshdata/streaming/_timeseries.py
@@ -12,22 +12,25 @@
needs — a single per-stream watermark timestamp — so a :class:`StreamingCleaner` can run
it batch-by-batch while still recognising events that arrive late relative to everything
seen so far. Every transformation is audited as a :class:`~freshdata.report.Action` with
-one of the ``timeseries_interpolation`` / ``seasonal_imputation`` / ``ordered_dedupe`` /
-``late_data`` / ``windowed_anomaly`` step names, so the trust contract is preserved.
+one of the ``timeseries_timestamp_parse`` / ``timeseries_interpolation`` /
+``seasonal_imputation`` / ``ordered_dedupe`` / ``late_data`` / ``windowed_anomaly`` step
+names, so the trust contract is preserved.
"""
from __future__ import annotations
+import warnings
from dataclasses import dataclass, field
from typing import Any
import numpy as np
import pandas as pd
-from .._util import safe_median
+from .._util import mask_sensitive_value, safe_median
from ..config import CleanConfig
from ..engine.context import infer_role
from ..report import CleanReport
+from ..steps.dtypes import COERCED_CELLS_CAP
#: Interpolation methods accepted by :class:`TimeSeriesCleanConfig`.
INTERPOLATION_METHODS = ("linear", "time", "ffill", "bfill")
@@ -39,6 +42,16 @@
ANOMALY_METHODS = ("rolling_zscore", "mad", "iqr", "ewma")
#: What to do with a flagged anomaly.
ANOMALY_ACTIONS = ("flag", "cap", "quarantine")
+#: Epoch units accepted for numeric timestamp columns.
+TIMESTAMP_UNITS = ("s", "ms", "us", "ns")
+#: Upper bounds on the median |epoch value| for each inferred unit: seconds up to
+#: 1e11 (year ~5100), milliseconds up to 1e14, microseconds up to 1e17, else ns.
+_EPOCH_UNIT_BOUNDS = ((1e11, "s"), (1e14, "ms"), (1e17, "us"))
+#: ``pd.to_datetime`` keyword sets :func:`coerce_datetimes` tries, in order.
+#: ``format="ISO8601"`` exists (and is needed) only on pandas >= 2.
+_DATETIME_PARSE_ATTEMPTS: tuple[dict[str, Any], ...] = (
+ ({}, {"utc": True}, {"format": "ISO8601"}, {"format": "ISO8601", "utc": True})
+ if int(pd.__version__.split(".")[0]) >= 2 else ({}, {"utc": True}))
#: Named seasonal buckets → a function mapping a datetime index to a season key.
_SEASON_KEYS = {
@@ -61,6 +74,95 @@ def to_timedelta(value: object) -> pd.Timedelta | None:
return pd.to_timedelta(value)
+def _n_lost(values: pd.Series, parsed: pd.Series) -> int:
+ """How many present values the parse turned into ``NaT``."""
+ return int((values.notna() & parsed.isna()).sum())
+
+
+def coerce_datetimes(values: pd.Series) -> pd.Series:
+ """Parse *values* with ``errors="coerce"``, surviving mixed UTC offsets.
+
+ Datetime columns pass through unchanged. Values with different UTC offsets
+ (for example either side of a DST change), or a mix of naive and
+ offset-aware values, have no single naive or fixed-offset representation.
+ Depending on the pandas version, a plain parse then raises, returns an
+ ``object`` column, or turns the minority into ``NaT``. So when the plain
+ parse is unusable or loses values, the batch is re-parsed with ``utc=True``
+ (reading naive values as UTC) and, on pandas >= 2, which infers one format
+ from the first value, also with ``format="ISO8601"``. The attempt that
+ loses the fewest values wins, the earliest on a tie, so clean batches pay
+ for one parse. Numeric input keeps pandas' default (nanoseconds);
+ :func:`parse_timestamps` infers epoch units.
+ """
+ if pd.api.types.is_datetime64_any_dtype(values):
+ return values
+ best: pd.Series | None = None
+ best_lost = 0
+ for kwargs in _DATETIME_PARSE_ATTEMPTS:
+ try:
+ with warnings.catch_warnings():
+ # pandas 2 warns before returning an object column for mixed
+ # offsets; a later utc=True attempt handles exactly that case.
+ warnings.filterwarnings("ignore", message=".*mixed time zones.*")
+ parsed = pd.to_datetime(values, errors="coerce", **kwargs)
+ except (ValueError, TypeError):
+ continue
+ if not pd.api.types.is_datetime64_any_dtype(parsed):
+ continue # object column of mixed-offset datetimes
+ lost = _n_lost(values, parsed)
+ if best is None or lost < best_lost:
+ best, best_lost = parsed, lost
+ if not lost:
+ break
+ if best is None: # every attempt raised: surface the plain parse's error
+ return pd.to_datetime(values, errors="coerce")
+ return best
+
+
+def infer_epoch_unit(values: pd.Series) -> str:
+ """Guess the epoch unit of numeric timestamps from their median magnitude."""
+ present = values.dropna()
+ if not len(present):
+ return "ns"
+ magnitude = float(present.abs().median())
+ for bound, unit in _EPOCH_UNIT_BOUNDS:
+ if magnitude < bound:
+ return unit
+ return "ns"
+
+
+def parse_timestamps(values: pd.Series, unit: str | None = None
+ ) -> tuple[pd.Series, str | None]:
+ """Parse a timestamp column; return ``(parsed, epoch_unit)``.
+
+ Numeric (non-bool) columns are epoch values in *unit*, or in the unit
+ :func:`infer_epoch_unit` infers when *unit* is ``None``; ``epoch_unit``
+ reports the unit used. Other columns go through :func:`coerce_datetimes`,
+ and ``epoch_unit`` is ``None``. Values that do not parse (or are out of
+ range) become ``NaT``.
+ """
+ dtype = values.dtype
+ if not (pd.api.types.is_numeric_dtype(dtype) and not pd.api.types.is_bool_dtype(dtype)):
+ return coerce_datetimes(values), None
+ unit = unit or infer_epoch_unit(values)
+ present = values.notna().to_numpy()
+ kept = values[present]
+ # Signed ints stay exact (ns epochs exceed float precision); everything else
+ # goes through float64 so nullable dtypes never hit pd.NA.
+ raw = (kept.to_numpy(dtype="int64") if pd.api.types.is_signed_integer_dtype(dtype)
+ else kept.to_numpy(dtype="float64"))
+ parsed = pd.Series(pd.NaT, index=values.index, dtype="datetime64[ns]")
+ if len(raw):
+ parsed[present] = pd.to_datetime(raw, unit=unit, errors="coerce").to_numpy()
+ return parsed, unit
+
+
+def _as_float(s: pd.Series) -> pd.Series:
+ """Score a column as float64; non-numeric cells (e.g. a stray string) become NaN."""
+ numeric = pd.to_numeric(s, errors="coerce")
+ return pd.Series(numeric.to_numpy(dtype="float64", na_value=np.nan), index=s.index)
+
+
@dataclass(frozen=True)
class TimeSeriesCleanConfig:
"""How a time-series / streaming-aware clean treats ordered, timestamped data.
@@ -118,6 +220,11 @@ class TimeSeriesCleanConfig:
Numeric column whose larger value wins ``highest_quality`` dedupe ties.
protected_columns:
Extra columns never interpolated, seasonally filled, or anomaly-scored.
+ timestamp_unit:
+ Epoch unit (one of :data:`TIMESTAMP_UNITS`) for *numeric* timestamp and
+ event-time columns. ``None`` (default) infers it from the magnitude of the
+ values (seconds, milliseconds, microseconds or nanoseconds). Ignored for
+ string and datetime columns.
"""
timestamp_column: str
@@ -139,6 +246,7 @@ class TimeSeriesCleanConfig:
anomaly_action: str = "flag"
quality_column: str | None = None
protected_columns: tuple[str, ...] = ()
+ timestamp_unit: str | None = None
def __post_init__(self) -> None:
if not isinstance(self.timestamp_column, str) or not self.timestamp_column:
@@ -150,8 +258,12 @@ def __post_init__(self) -> None:
_check("anomaly_action", self.anomaly_action, ANOMALY_ACTIONS)
if self.max_interpolation_gap < 0:
raise ValueError("max_interpolation_gap must be >= 0")
- if self.anomaly_window_size < 0:
- raise ValueError("anomaly_window_size must be >= 0")
+ if self.anomaly_window_size < 0 or self.anomaly_window_size == 1:
+ # A one-row window has no spread to score against (and the rolling
+ # min_periods of 2 would exceed it), so 0 (disabled) or >= 2 only.
+ raise ValueError("anomaly_window_size must be 0 or >= 2")
+ if self.timestamp_unit is not None:
+ _check("timestamp_unit", self.timestamp_unit, TIMESTAMP_UNITS)
if self.anomaly_threshold <= 0:
raise ValueError("anomaly_threshold must be > 0")
if (self.ordered_dedupe_keep == "highest_quality"
@@ -226,7 +338,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *,
# Nothing time-series-shaped about this batch; leave it untouched.
self.last_summary = {}
return df, _empty_like(df)
- df[ts_col] = pd.to_datetime(df[ts_col], errors="coerce")
+ df[ts_col] = self._parse_timestamp_column(df[ts_col], report)
# 1. Late data is judged in arrival order, before any sort reorders the batch.
df, late_exc, late_meta = self._handle_late_data(df, report)
@@ -244,6 +356,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *,
roles = roles or self._infer_roles(df)
numeric_cols = self.numeric_targets(df, roles)
self.last_numeric_cols = [str(c) for c in numeric_cols]
+ anomaly_cols = self.anomaly_targets(df, roles)
# 3. Short-gap interpolation, then 4. seasonal imputation of what's left.
df = self._interpolate(df, numeric_cols, report)
@@ -251,7 +364,7 @@ def process(self, df: pd.DataFrame, report: CleanReport, *,
df = self._seasonal_impute(df, numeric_cols, report)
# 5. Windowed anomaly detection (flag columns by default; cap/quarantine opt-in).
- df, anom_exc, anom_meta = self._windowed_anomaly(df, numeric_cols, report)
+ df, anom_exc, anom_meta = self._windowed_anomaly(df, anomaly_cols, report)
if anom_exc is not None and len(anom_exc):
cleaned_exc.append(anom_exc)
summary.update(anom_meta)
@@ -263,6 +376,49 @@ def process(self, df: pd.DataFrame, report: CleanReport, *,
self.last_summary = summary
return df, exceptions
+ def _parse_timestamp_column(self, raw: pd.Series, report: CleanReport) -> pd.Series:
+ """Parse the timestamp column, auditing the epoch unit and every lost value.
+
+ Rows whose timestamp does not parse are kept (with ``NaT``); their
+ original values are preserved in ``report.coerced_cells``.
+ """
+ cfg = self.config
+ name = str(cfg.timestamp_column)
+ parsed, epoch_unit = parse_timestamps(raw, cfg.timestamp_unit)
+ if epoch_unit is not None:
+ source = ("timestamp_unit" if cfg.timestamp_unit
+ else "inferred from the magnitude of the values")
+ report.add("timeseries_timestamp_parse",
+ f"read numeric timestamps as epoch unit '{epoch_unit}'",
+ column=name, count=int(parsed.notna().sum()), risk="low",
+ rationale=f"epoch unit {source}")
+ lost = raw.notna() & parsed.isna()
+ n_lost = int(lost.sum())
+ if not n_lost:
+ return parsed
+ originals = raw[lost]
+ sensitive = name in self.clean_config.sensitive_columns
+ cells = report.coerced_cells.setdefault(name, {})
+ for row, value in originals.items():
+ if len(cells) >= COERCED_CELLS_CAP:
+ break
+ cells[row] = mask_sensitive_value(value) if sensitive else value
+ report.coerced_rows[name] = tuple(
+ dict.fromkeys([*report.coerced_rows.get(name, ()), *originals.index]))
+ examples = ", ".join(
+ f"{mask_sensitive_value(v) if sensitive else repr(v)} (row {i})"
+ for i, v in list(originals.head(3).items()))
+ report.add("timeseries_timestamp_parse",
+ f"{n_lost} timestamp value(s) could not be parsed and were set "
+ "to missing; rows kept", column=name, count=n_lost, risk="medium",
+ rationale="unparseable or out-of-range timestamp; originals "
+ "preserved in report.coerced_cells")
+ report.add_warning(
+ f"column '{name}': {n_lost} timestamp value(s) could not be parsed "
+ f"and were set to missing, e.g. {examples}. Originals are preserved "
+ "in report.coerced_cells.")
+ return parsed
+
# -- step 5 (numbered by the spec): watermark-aware late data ---------------
def _handle_late_data(self, df: pd.DataFrame, report: CleanReport
@@ -274,7 +430,7 @@ def _handle_late_data(self, df: pd.DataFrame, report: CleanReport
event_col = cfg.resolved_event_time_column
if event_col not in df.columns:
return df, None, {}
- event_time = pd.to_datetime(df[event_col], errors="coerce")
+ event_time, _ = parse_timestamps(df[event_col], cfg.timestamp_unit)
late_mask = self._late_mask(df, event_time, lateness)
n_late = int(late_mask.sum())
@@ -512,7 +668,7 @@ def _windowed_anomaly(self, df: pd.DataFrame, numeric_cols: list[str],
def score_group(g: pd.DataFrame, col: str = col, flags: pd.Series = flags,
lower_all: pd.Series = lower_all,
upper_all: pd.Series = upper_all) -> pd.DataFrame:
- s = g[col].astype("float64")
+ s = _as_float(g[col])
f, lo, hi = _anomaly_scores(s, win, thr, method)
flags.loc[g.index] = f.to_numpy()
lower_all.loc[g.index] = lo.to_numpy()
@@ -529,7 +685,7 @@ def score_group(g: pd.DataFrame, col: str = col, flags: pd.Series = flags,
self.anomalies_flagged_total += n_flag
action_note = "flagged"
if cfg.anomaly_action == "cap":
- capped = df[col].astype("float64").clip(lower=lower_all, upper=upper_all)
+ capped = _as_float(df[col]).clip(lower=lower_all, upper=upper_all)
df.loc[flags, col] = capped[flags]
report.outliers_handled += n_flag
action_note = "flagged and capped"
@@ -579,6 +735,26 @@ def numeric_targets(self, df: pd.DataFrame,
impute, so the time-series policy (short-gap interpolation, long-gap
preservation) owns their missing values.
"""
+ return [c for c in self._numeric_role_columns(df, roles)
+ if pd.api.types.is_numeric_dtype(df[c])]
+
+ def anomaly_targets(self, df: pd.DataFrame,
+ roles: dict[str, str] | None = None) -> list[str]:
+ """Columns that get a ``_anomaly`` flag column.
+
+ Chosen by (locked) role rather than by this batch's dtype: a numeric
+ column whose batch holds a stray string arrives as ``object``, and must
+ still be scored (non-numeric cells score as missing) so every batch emits
+ the same flag columns.
+ """
+ return [c for c in self._numeric_role_columns(df, roles)
+ if pd.api.types.is_numeric_dtype(df[c])
+ or pd.api.types.is_object_dtype(df[c])
+ or pd.api.types.is_string_dtype(df[c])]
+
+ def _numeric_role_columns(self, df: pd.DataFrame,
+ roles: dict[str, str] | None) -> list[Any]:
+ """Unprotected columns whose role is ``numeric``, as the frame's own labels."""
roles = roles or self._infer_roles(df)
cfg = self.config
protected = {
@@ -593,8 +769,6 @@ def numeric_targets(self, df: pd.DataFrame,
continue
if roles.get(name) != "numeric":
continue
- if not pd.api.types.is_numeric_dtype(df[c]):
- continue
cols.append(c) # the frame's own label (may be an int), not str(c)
return cols
@@ -614,8 +788,16 @@ def _anomaly_scores(s: pd.Series, win: int, thr: float, method: str
med = s.rolling(win, min_periods=min_p).median()
mad = (s - med).abs().rolling(win, min_periods=min_p).median()
robust_z = 0.6745 * (s - med) / mad.replace(0, np.nan)
- lower, upper = med - thr * mad / 0.6745, med + thr * mad / 0.6745
- flags = _flag(robust_z, thr, s, med, mad)
+ # A zero MAD only means more than half the window equals the median,
+ # which is routine for sparse or two-valued series. The window is flat
+ # only when every *other* point in it (the win - 1 before this one) is
+ # identical; a deviating point there is a spike. Otherwise a zero MAD
+ # gives no scale, so the point is not flagged and has no fence to cap to.
+ others = s.shift(1).rolling(win - 1, min_periods=1)
+ flat = (mad == 0) & (others.max() == others.min())
+ fence = mad.where((mad > 0) | flat)
+ lower, upper = med - thr * fence / 0.6745, med + thr * fence / 0.6745
+ flags = _flag(robust_z, thr, s, med, flat)
return flags, lower, upper
if method == "ewma":
# Judge each point against the EWMA *forecast* from prior points (shifted by one).
@@ -625,24 +807,25 @@ def _anomaly_scores(s: pd.Series, win: int, thr: float, method: str
std = s.ewm(span=win, min_periods=min_p).std().shift(1)
z = (s - mean) / std.replace(0, np.nan)
lower, upper = mean - thr * std, mean + thr * std
- flags = _flag(z, thr, s, mean, std)
+ flags = _flag(z, thr, s, mean, std == 0)
return flags, lower, upper
# rolling_zscore (default)
mean = s.rolling(win, min_periods=min_p).mean()
std = s.rolling(win, min_periods=min_p).std()
z = (s - mean) / std.replace(0, np.nan)
lower, upper = mean - thr * std, mean + thr * std
- flags = _flag(z, thr, s, mean, std)
+ flags = _flag(z, thr, s, mean, std == 0)
return flags, lower, upper
def _flag(z: pd.Series, thr: float, s: pd.Series, center: pd.Series,
- scale: pd.Series) -> pd.Series:
+ flat: pd.Series) -> pd.Series:
"""Flag points beyond *thr* standard scores, plus the degenerate case where the
- rolling scale is exactly zero (a flat window) yet the point still deviates from the
- centre — there the z-score is NaN, so it would otherwise slip through unflagged."""
+ window is *flat* (the caller decides what that means for its scale) yet the point
+ still deviates from the centre — there the z-score is NaN, so it would otherwise
+ slip through unflagged."""
beyond = (z.abs() > thr).fillna(False)
- flat_spike = (scale.fillna(np.nan) == 0) & ((s - center).abs() > 0)
+ flat_spike = flat.fillna(False) & ((s - center).abs() > 0)
return (beyond | flat_spike.fillna(False))
diff --git a/tests/test_cdc_profile.py b/tests/test_cdc_profile.py
index 670ea132..18846d4c 100644
--- a/tests/test_cdc_profile.py
+++ b/tests/test_cdc_profile.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import time
+
import pandas as pd
import pytest
@@ -154,3 +156,56 @@ def test_does_not_mutate_input():
before = df.copy(deep=True)
fd.cdc_profile(df, event_time="event_ts", key="entity_id", now=NOW)
pd.testing.assert_frame_equal(df, before)
+
+
+# -- time zones (#233 part 4) ----------------------------------------------------
+
+AWARE = _events(ts=pd.to_datetime(["2024-01-01 00:10", "2024-01-01 00:00"]).tz_localize("UTC"))
+
+
+def test_default_now_is_utc_for_naive_event_times(monkeypatch):
+ if hasattr(time, "tzset"): # simulate a host west of UTC
+ monkeypatch.setenv("TZ", "America/New_York")
+ time.tzset()
+ try:
+ event = pd.Timestamp.now(tz="UTC").tz_localize(None) - pd.Timedelta("3h")
+ rep = fd.cdc_profile(_events(ts=[event]), event_time="ts", stale_after="1h")
+ finally:
+ monkeypatch.undo()
+ if hasattr(time, "tzset"):
+ time.tzset()
+ assert rep.freshness_seconds == pytest.approx(3 * 3600, abs=60)
+ assert any(d.kind == "stale" for d in rep.defects)
+
+
+def test_default_now_with_aware_event_times():
+ event = pd.Timestamp.now(tz="UTC") - pd.Timedelta("2h")
+ rep = fd.cdc_profile(_events(ts=[event.tz_convert("Asia/Kolkata")]), event_time="ts")
+ assert rep.freshness_seconds == pytest.approx(2 * 3600, abs=60)
+
+
+def test_naive_now_with_aware_event_times_is_read_as_utc():
+ rep = fd.cdc_profile(AWARE, event_time="ts", now="2024-01-02")
+ assert rep.freshness_seconds == pytest.approx(24 * 3600 - 600)
+
+
+def test_aware_now_with_naive_event_times_is_converted_to_utc():
+ df = _events(ts=pd.to_datetime(["2024-01-01 10:00"]))
+ rep = fd.cdc_profile(df, event_time="ts", now="2024-01-01T12:00:00+01:00")
+ assert rep.freshness_seconds == pytest.approx(3600)
+
+
+def test_naive_watermark_with_aware_event_times():
+ rep = fd.cdc_profile(AWARE, event_time="ts", watermark="2024-01-01 00:05", now=NOW)
+ late = [d for d in rep.defects if d.kind == "late"]
+ assert late and late[0].n_rows == 1
+
+
+def test_mixed_utc_offsets_are_normalised():
+ mixed = _events(ts=["2024-03-10T01:30:00-05:00", "2024-03-10T03:30:00-04:00",
+ "2024-03-10T02:00:00-05:00"], k=["a", "a", "a"])
+ rep = fd.cdc_profile(mixed, event_time="ts", key="k",
+ now=pd.Timestamp("2024-03-11", tz="UTC"))
+ # 06:30Z, 07:30Z, then 07:00Z: only the last row is behind the running max.
+ assert {d.kind: d.n_rows for d in rep.defects} == {"late": 1}
+ assert rep.freshness_seconds == pytest.approx(16.5 * 3600)
diff --git a/tests/test_streaming_timeseries.py b/tests/test_streaming_timeseries.py
index caf7545d..23885a05 100644
--- a/tests/test_streaming_timeseries.py
+++ b/tests/test_streaming_timeseries.py
@@ -11,7 +11,8 @@
import freshdata as fd
from freshdata import StreamingCleaner, TimeSeriesCleanConfig
-from freshdata.streaming._timeseries import to_timedelta
+from freshdata.report import CleanReport
+from freshdata.streaming._timeseries import TimeSeriesProcessor, coerce_datetimes, to_timedelta
# -- config validation ---------------------------------------------------------
@@ -347,3 +348,171 @@ def test_integer_column_labels_in_timeseries_mode():
assert 7 in out.columns and 8 in out.columns
assert out[8].tolist() == [1.0, 2.0, 3.0, 4.0]
assert any(a.step == "timeseries_interpolation" and a.column == "8" for a in report)
+
+
+# -- timestamp parsing (#227, #250) ---------------------------------------------
+
+def _ts_clean(df, **kw):
+ cfg = TimeSeriesCleanConfig(timestamp_column="ts", **kw)
+ return StreamingCleaner(time_series_config=cfg, verbose=False).clean_batch(df)
+
+
+@pytest.mark.parametrize("scale,unit", [(1, "s"), (10**3, "ms"), (10**6, "us"), (10**9, "ns")])
+def test_integer_epoch_timestamps_infer_unit(scale, unit):
+ epoch = [(1704067200 + 3600 * i) * scale for i in range(4)]
+ df = pd.DataFrame({"ts": epoch, "v": [1.0, 2.0, 3.0, 4.0]})
+ out, report = _ts_clean(df)
+ assert out["ts"].tolist() == list(pd.date_range("2024-01-01", periods=4, freq="h"))
+ parse = [a for a in report.actions if a.step == "timeseries_timestamp_parse"]
+ assert [a.column for a in parse] == ["ts"]
+ assert f"'{unit}'" in parse[0].description
+ assert not report.coerced_cells
+
+
+def test_float_and_nullable_epoch_timestamps_keep_rows():
+ for values in ([1704067200.0, np.nan, 1704074400.0],
+ pd.array([1704067200, None, 1704074400], dtype="Int64")):
+ df = pd.DataFrame({"ts": values, "v": [1.0, 2.0, 3.0]})
+ out, report = _ts_clean(df)
+ assert len(out) == 3
+ assert out["ts"].iloc[0] == pd.Timestamp("2024-01-01 00:00")
+ assert not report.coerced_cells # a missing epoch is not a lost value
+
+
+def test_explicit_timestamp_unit_overrides_inference():
+ df = pd.DataFrame({"ts": [1704067200, 1704067260], "v": [1.0, 2.0]})
+ out, _ = _ts_clean(df, timestamp_unit="ms")
+ assert out["ts"].iloc[0] == pd.Timestamp(1704067200, unit="ms")
+ with pytest.raises(ValueError, match="timestamp_unit"):
+ TimeSeriesCleanConfig(timestamp_column="ts", timestamp_unit="minutes")
+
+
+def test_unparseable_timestamps_are_reported_and_rows_kept():
+ df = pd.DataFrame({"ts": ["2024-01-01", "2024-13-45", "2024-01-03"],
+ "v": [1.0, 2.0, 3.0]})
+ out, report = _ts_clean(df)
+ assert len(out) == 3
+ assert out["ts"].isna().sum() == 1
+ assert report.coerced_cells == {"ts": {1: "2024-13-45"}}
+ assert report.coerced_rows["ts"] == (1,)
+ assert report.to_dict()["coerced_cells"] == {"ts": {"1": "2024-13-45"}}
+ lost = [a for a in report.actions
+ if a.step == "timeseries_timestamp_parse" and a.count == 1]
+ assert lost and lost[0].risk == "medium"
+ assert any("2024-13-45" in w for w in report.warnings)
+
+
+def test_out_of_range_epoch_is_reported():
+ df = pd.DataFrame({"ts": [1704067200, 99_999_999_999], "v": [1.0, 2.0]})
+ out, report = _ts_clean(df, timestamp_unit="s")
+ assert out["ts"].isna().sum() == 1
+ assert list(report.coerced_cells["ts"].values()) == [99_999_999_999]
+
+
+def test_unparseable_sensitive_timestamp_is_masked_in_report():
+ proc =TimeSeriesProcessor(TimeSeriesCleanConfig(timestamp_column="ts"),
+ clean_config=fd.CleanConfig(sensitive_columns=("ts",)))
+ report = CleanReport()
+ df = pd.DataFrame({"ts": ["2024-01-01", "secret-value"], "v": [1.0, 2.0]})
+ proc.process(df, report)
+ (masked,) = report.coerced_cells["ts"].values()
+ assert masked.startswith("[SENSITIVE:")
+ assert not any("secret-value" in w for w in report.warnings)
+
+
+def test_mixed_utc_offsets_across_dst_are_parsed_to_utc():
+ df = pd.DataFrame({"ts": ["2024-03-10T01:00:00-05:00", "2024-03-10T01:30:00-05:00",
+ "2024-03-10T03:00:00-04:00", "2024-03-10T03:30:00-04:00"],
+ "v": [1.0, None, 3.0, 4.0]})
+ out, report = _ts_clean(df)
+ assert str(out["ts"].dt.tz) == "UTC"
+ assert out["ts"].tolist() == list(
+ pd.date_range("2024-03-10 06:00", periods=4, freq="30min", tz="UTC"))
+ assert out["v"].tolist() == pytest.approx([1.0, 2.0, 3.0, 4.0])
+ assert not report.coerced_cells
+
+
+def test_naive_and_offset_aware_timestamps_mix_parses_to_utc():
+ df = pd.DataFrame({"ts": ["2024-03-10T01:00:00", "2024-03-10T03:00:00-04:00"],
+ "v": [1.0, 2.0]})
+ out, report = _ts_clean(df)
+ assert out["ts"].tolist() == [pd.Timestamp("2024-03-10 01:00", tz="UTC"),
+ pd.Timestamp("2024-03-10 07:00", tz="UTC")]
+ assert not report.coerced_cells
+
+
+def test_coerce_datetimes_keeps_clean_batches_in_their_own_zone():
+ naive = pd.Series(["2024-01-01 00:00", "2024-01-02 00:00", None])
+ assert coerce_datetimes(naive).dt.tz is None
+ one_offset = coerce_datetimes(pd.Series(["2024-01-01T00:00:00+05:30"] * 2))
+ assert one_offset.iloc[0] == pd.Timestamp("2023-12-31 18:30", tz="UTC")
+ assert str(one_offset.dt.tz) != "UTC" # a single fixed offset is kept
+ garbage = coerce_datetimes(pd.Series(["2024-01-01", "not a date"]))
+ assert garbage.dt.tz is None and garbage.isna().tolist() == [False, True]
+
+
+def test_epoch_event_time_column_drives_late_data():
+ df = pd.DataFrame({"ts": pd.date_range("2024", periods=3, freq="h"),
+ "ev": [1704067200, 1704070800, 1704067210], "v": [1.0, 2.0, 3.0]})
+ out, report = _ts_clean(df, event_time_column="ev", allowed_lateness="1m")
+ assert len(out) == 2
+ assert any(a.step == "late_data" and a.count == 1 for a in report.actions)
+
+
+# -- anomaly config and MAD (#290, #291) ----------------------------------------
+
+def test_anomaly_window_size_one_is_rejected():
+ with pytest.raises(ValueError, match="0 or >= 2"):
+ TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=1)
+ df = pd.DataFrame({"ts": pd.date_range("2024", periods=5, freq="h"),
+ "v": [1.0, 2.0, 3.0, 4.0, 5.0]})
+ out, _ = _ts_clean(df, anomaly_window_size=2)
+ assert "v_anomaly" in out.columns
+
+
+def test_mad_does_not_flag_minority_values_of_two_valued_series():
+ v = [0.0, 0, 0, 1, 0, 0, 1, 0, 0, 0] * 3
+ df = pd.DataFrame({"ts": pd.date_range("2024", periods=30, freq="h"), "v": v})
+ out, _ = _ts_clean(df, anomaly_window_size=10, anomaly_method="mad",
+ anomaly_action="cap")
+ assert int(out["v_anomaly"].sum()) == 0
+ assert out["v"].sum() == 6
+
+
+def test_mad_still_flags_and_caps_spike_in_flat_window():
+ x = np.full(40, 5.0)
+ x[25] = 999.0
+ df = pd.DataFrame({"ts": pd.date_range("2024", periods=40, freq="min"), "x": x})
+ out, _ = _ts_clean(df, anomaly_window_size=10, anomaly_method="mad",
+ anomaly_action="cap")
+ assert out["x_anomaly"].tolist() == [i == 25 for i in range(40)]
+ assert out["x"].iloc[25] == 5.0
+
+
+# -- anomaly columns when batch dtypes change (#248 part b) ----------------------
+
+def test_anomaly_columns_stable_when_numeric_column_holds_a_string():
+ cfg = TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=20,
+ anomaly_method="mad")
+ cleaner = StreamingCleaner(time_series_config=cfg, verbose=False)
+ b1 = pd.DataFrame({"ts": ["2024-01-01", "2024-01-02", "2024-01-03"],
+ "a": [0, 1, 2], "b": [0, 10, 20]})
+ b2 = pd.DataFrame({"ts": ["2024-01-04", "2024-01-05", "2024-01-06"],
+ "a": ["oops", 4, 5], "b": [30, 40, 50]})
+ out1, _ = cleaner.clean_batch(b1)
+ out2, _ = cleaner.clean_batch(b2)
+ assert list(out2.columns) == list(out1.columns) == ["ts", "a", "b", "a_anomaly", "b_anomaly"]
+ assert out2["a"].tolist() == ["oops", 4, 5]
+
+
+def test_non_numeric_cell_is_neither_scored_nor_capped():
+ cfg = TimeSeriesCleanConfig(timestamp_column="ts", anomaly_window_size=5,
+ anomaly_method="mad", anomaly_action="cap")
+ cleaner = StreamingCleaner(time_series_config=cfg, verbose=False)
+ cleaner.clean_batch(pd.DataFrame({"ts": pd.date_range("2024-01", periods=8, freq="h"),
+ "a": [5.0] * 8})) # locks 'a' as numeric
+ b2 = pd.DataFrame({"ts": pd.date_range("2024-02", periods=8, freq="h"),
+ "a": [5, 5, 5, 5, "oops", 5, 999, 5]})
+ out, _ = cleaner.clean_batch(b2)
+ assert out["a_anomaly"].tolist() == [i == 6 for i in range(8)]
+ assert out["a"].tolist() == [5, 5, 5, 5, "oops", 5, 5.0, 5]