diff --git a/src/freshdata/enterprise/contracts.py b/src/freshdata/enterprise/contracts.py index 52c81763..34acfd0c 100644 --- a/src/freshdata/enterprise/contracts.py +++ b/src/freshdata/enterprise/contracts.py @@ -88,6 +88,21 @@ def _round(value: float | None, ndigits: int = 6) -> float | None: return round(float(value), ndigits) +def _full_precision(value: float | None) -> float | None: + """*value* as an unrounded float (JSON round-trips it exactly); ``None`` if NaN/inf. + + Baseline statistics are stored this way: rounding to decimal places erases + small-magnitude columns, and any rounding makes stored quantiles differ from + the data values they came from, which the drift checks compare exactly. + """ + if value is None: + return None + v = float(value) + if math.isnan(v) or math.isinf(v): + return None + return v + + def _hash_label(value: str) -> str: """Stable, non-reversible category label for PII-safe baselines.""" return "h:" + hashlib.sha1(value.encode("utf-8")).hexdigest()[:16] @@ -99,19 +114,26 @@ def _normalize_dtype(dtype: str | None) -> str | None: ``int64``/``Int32`` → ``int``; ``float64`` → ``float``; ``object``/``string`` → ``string``; ``datetime64[ns, UTC]`` → ``datetime``; ``bool`` → ``bool``. Keeps drift comparison robust to width and nullable-extension variants. + ``pd.ArrowDtype`` names (``double[pyarrow]``, ``large_string[pyarrow]``, + ``decimal128(10, 2)[pyarrow]``, ``timestamp[us, tz=UTC][pyarrow]``, …) map + to the same families. """ if dtype is None: return None d = dtype.strip().lower() + if d.endswith("[pyarrow]"): + d = d[: -len("[pyarrow]")] if d.startswith(("int", "uint")) or d in ("int", "integer"): return "int" - if d.startswith("float") or d in ("double", "decimal"): + if d.startswith(("float", "halffloat", "decimal")) or d == "double": return "float" if d.startswith("bool"): return "bool" + if d.startswith("dictionary"): # Arrow's categorical encoding + return "string" if d.startswith(("datetime", "timestamp")) or "date" in d: return "datetime" - if d.startswith(("object", "string", "str", "category", "utf8")): + if d.startswith(("object", "string", "str", "category", "utf8", "large_string", "large_utf8")): return "string" return d @@ -316,11 +338,11 @@ def to_dict(self) -> dict[str, Any]: "n_unique": self.n_unique, "n_rows": self.n_rows, "sample_values": list(self.sample_values), - "min": _round(self.min), - "max": _round(self.max), - "mean": _round(self.mean), - "std": _round(self.std), - "quantiles": {k: _round(v) for k, v in self.quantiles.items()}, + "min": _full_precision(self.min), + "max": _full_precision(self.max), + "mean": _full_precision(self.mean), + "std": _full_precision(self.std), + "quantiles": {k: _full_precision(v) for k, v in self.quantiles.items()}, "top_values": list(self.top_values), "frequencies": {k: _round(v) for k, v in self.frequencies.items()}, "min_timestamp": self.min_timestamp, @@ -702,7 +724,8 @@ def _profile_column(series: pd.Series, *, n_rows: int, include_samples: bool) -> cb.sample_values = tuple(str(v) for v in uniques) if family in ("int", "float") and len(non_null): - numeric = pd.to_numeric(non_null, errors="coerce").dropna() + # float: Arrow decimals come back as Decimal objects, which can't take std(). + numeric = pd.to_numeric(non_null, errors="coerce").dropna().astype(float) if len(numeric): cb.min = float(numeric.min()) cb.max = float(numeric.max()) @@ -803,20 +826,66 @@ def _ks_statistic(cb: ColumnBaseline, current: pd.Series) -> float | None: n = len(vals) if n == 0: return None + knots, p_lo, p_hi = _cdf_knots(pts) + cur = np.sort(vals) + + # At a stored value x the quantiles only bound the baseline CDF: + # F(x-) <= p_lo and F(x) >= p_hi. The jump F(x) - F(x-) is a point mass, + # which can be no larger than the baseline's distinct-value count allows. + # Measure the distance to the nearest baseline CDF within those bounds, so + # ties (discrete, zero-inflated or constant columns) don't read as drift. + mass = _max_point_mass(cb) + below = np.searchsorted(cur, knots, side="left") / n + at = np.searchsorted(cur, knots, side="right") / n + at_knots = np.maximum.reduce( + [ + below - p_lo, + np.minimum(p_hi - mass, p_lo) - below, + p_hi - at, + at - np.maximum(p_lo + mass, p_hi), + ] + ) + stat = max(0.0, float(np.max(at_knots))) + + # Between knots the baseline CDF is interpolated linearly, as before. + grid = np.quantile(vals, [p for p, _ in pts]) + grid = grid[~np.isin(grid, knots)] + if len(grid): + f_base = _cdf_between_knots(grid, knots, p_lo, p_hi) + f_cur = np.searchsorted(cur, grid, side="right") / n + stat = max(stat, float(np.max(np.abs(f_base - f_cur)))) + return stat + + +def _cdf_knots(pts: list[tuple[float, float]]) -> tuple[np.ndarray, np.ndarray, np.ndarray]: + """Distinct knot values with the lowest and highest probability stored at each.""" xs = np.array([v for _, v in pts], dtype=float) ps = np.array([p for p, _ in pts], dtype=float) - # Ensure strictly increasing xs for interpolation. order = np.argsort(xs, kind="stable") xs, ps = xs[order], ps[order] - keep = np.concatenate(([True], np.diff(xs) > 0)) - xs, ps = xs[keep], ps[keep] - if len(xs) < 2: - return None - - grid = np.unique(np.concatenate([xs, np.quantile(vals, ps)])) - f_base = np.interp(grid, xs, ps, left=0.0, right=1.0) - f_cur = np.searchsorted(np.sort(vals), grid, side="right") / n - return float(np.max(np.abs(f_base - f_cur))) + knots, first = np.unique(xs, return_index=True) + return knots, np.minimum.reduceat(ps, first), np.maximum.reduceat(ps, first) + + +def _max_point_mass(cb: ColumnBaseline) -> float: + """Largest share of non-null baseline rows a single value can hold.""" + n = cb.n_rows * (1.0 - (cb.missing_ratio or 0.0)) + if n < 1 or cb.n_unique < 1: + return 1.0 + return min(1.0, max(0.0, (n - cb.n_unique + 1) / n)) + + +def _cdf_between_knots( + x: np.ndarray, knots: np.ndarray, p_lo: np.ndarray, p_hi: np.ndarray +) -> np.ndarray: + """Baseline CDF at points *x* that are not knots: linear between, 0/1 outside.""" + idx = np.searchsorted(knots, x, side="right") - 1 + out = np.where(idx < 0, 0.0, 1.0) + inside = (idx >= 0) & (idx < len(knots) - 1) + i = idx[inside] + frac = (x[inside] - knots[i]) / (knots[i + 1] - knots[i]) + out[inside] = p_hi[i] + frac * (p_lo[i + 1] - p_hi[i]) + return out def _psi_numeric(cb: ColumnBaseline, current: pd.Series) -> float | None: @@ -828,28 +897,21 @@ def _psi_numeric(cb: ColumnBaseline, current: pd.Series) -> float | None: n = len(vals) if n == 0: return None - probs = [p for p, _ in pts] - edges = [v for _, v in pts] - # Merge bins whose edges collapse (repeated quantile values). - merged_edges: list[float] = [edges[0]] - merged_expected: list[float] = [] - acc = 0.0 - for i in range(1, len(edges)): - acc += probs[i] - probs[i - 1] - if edges[i] > merged_edges[-1]: - merged_edges.append(edges[i]) - merged_expected.append(acc) - acc = 0.0 - if acc > 0 and merged_expected: - merged_expected[-1] += acc - if len(merged_edges) < 2: + knots, p_lo, _ = _cdf_knots(pts) + if len(knots) < 2: return None - inner = np.array(merged_edges[1:-1], dtype=float) - idx = np.searchsorted(inner, vals, side="right") if len(inner) else np.zeros(n, dtype=int) - actual_counts = np.bincount(idx, minlength=len(merged_expected)).astype(float) - actual = actual_counts / n - expected = np.array(merged_expected, dtype=float) - expected = expected / expected.sum() if expected.sum() else expected + # Bins run from one distinct quantile value to the next; repeated values + # collapse into one edge. The expected share below an edge is the lowest + # probability stored at it. Current values equal to an edge are split + # across it the same way, so a point mass on a quantile isn't counted + # wholly in the upper bin. + inner = knots[1:-1] + cut = p_lo[1:-1] + cur = np.sort(vals) + below = np.searchsorted(cur, inner, side="left") / n + at = np.searchsorted(cur, inner, side="right") / n + expected = np.diff(np.concatenate(([0.0], cut, [1.0]))) + actual = np.diff(np.concatenate(([0.0], np.clip(cut, below, at), [1.0]))) return _psi(expected, actual) diff --git a/tests/test_baseline_compare.py b/tests/test_baseline_compare.py index db145c2f..a7278561 100644 --- a/tests/test_baseline_compare.py +++ b/tests/test_baseline_compare.py @@ -2,7 +2,9 @@ from __future__ import annotations +import numpy as np import pandas as pd +import pytest import freshdata as fd from freshdata import build_baseline @@ -66,3 +68,109 @@ def test_prebuilt_baseline_still_works() -> None: bl = build_baseline(base, name="weekly") diff = fd.compare_to_baseline(base, bl) # positional, original API assert diff.passed + + +# --------------------------------------------------------------------------- +# Distribution drift maths: ties, constant columns and saved precision +# (#234, #235, #275) +# --------------------------------------------------------------------------- + + +def _drift_problems(report: fd.DriftReport) -> list[tuple[str | None, str, str]]: + return [ + (f.check_id, f.status, f.message) + for f in report.findings + if (f.check_id or "").startswith("drift.") and f.status != "passed" + ] + + +_rs = np.random.RandomState(3) +_TIED_COLUMNS = { + "zero_inflated_counts": [0] * 99 + [1], + "small_int_ratings": [1, 2, 3] * 40, + "constant": [5.0] * 100, + # The median lands on a value that holds most of the rows between p25 and p75. + "atom_straddling_quantiles": [0] * 9 + [1] * 15 + [2] * 8, + "zero_inflated_continuous": list(np.where(_rs.rand(500) < 0.6, 0.0, _rs.exponential(5, 500))), + "thirds": [k / 3 for k in (0, 1, 2)] * 40, + "inexact_tenths": [0.1 * k for k in (1, 2, 3)] * 40, # 0.30000000000000004 +} + + +def test_issue_234_repro_frames_match_their_own_baseline() -> None: + df = pd.DataFrame({"refunds": [0] * 99 + [1]}) + rep = fd.compare_to_baseline(df, fd.build_baseline(df, name="self")) + df2 = pd.DataFrame({"rating": [1, 2, 3] * 40}) + rep2 = fd.compare_to_baseline(df2, fd.build_baseline(df2, name="self")) + assert rep.passed and rep2.passed + assert _drift_problems(rep) == [] and _drift_problems(rep2) == [] + + +@pytest.mark.parametrize("values", list(_TIED_COLUMNS.values()), ids=list(_TIED_COLUMNS)) +def test_frame_with_tied_values_matches_its_own_baseline(values: list[float], tmp_path) -> None: + df = pd.DataFrame({"x": values}) + base = build_baseline(df, name="self") + fd.save_baseline(base, tmp_path / "self.json") + for bl in (base, fd.load_baseline(tmp_path / "self.json")): + report = fd.compare_to_baseline(df, bl) + assert report.passed + assert _drift_problems(report) == [] + + +def test_tied_baseline_still_flags_real_drift() -> None: + refunds = build_baseline(pd.DataFrame({"refunds": [0] * 99 + [1]}), name="b") + more_refunds = pd.DataFrame({"refunds": [0] * 80 + [1] * 20}) + assert not fd.compare_to_baseline(more_refunds, refunds).passed + ratings = build_baseline(pd.DataFrame({"rating": [1, 2, 3] * 40}), name="b") + assert not fd.compare_to_baseline(pd.DataFrame({"rating": [3] * 120}), ratings).passed + + +def test_point_mass_on_a_continuous_baseline_quantile_is_still_drift() -> None: + # e.g. missing values imputed with the median: the baseline had no such spike. + rs = np.random.RandomState(0) + values = rs.normal(0, 1, 2000) + current = values.copy() + current[:1200] = np.median(values) + base = build_baseline(pd.DataFrame({"x": values}), name="b") + report = fd.compare_to_baseline(pd.DataFrame({"x": current}), base) + assert ("drift.ks", "failed") in {(f.check_id, f.status) for f in report.findings} + + +def test_constant_baseline_column_still_checks_distribution() -> None: + base = build_baseline(pd.DataFrame({"price": [5.0] * 100}), name="b") + + shifted = fd.compare_to_baseline(pd.DataFrame({"price": [1000.0] * 100}), base) + assert not shifted.passed + ks = [f for f in shifted.findings if f.check_id == "drift.ks"] + assert [(f.status, f.current_value) for f in ks] == [("failed", 1.0)] + assert shifted.distribution_drift["price"]["ks"] == 1.0 + + partly = fd.compare_to_baseline(pd.DataFrame({"price": [4.0] * 20 + [5.0] * 80}), base) + assert [(f[0], f[1]) for f in _drift_problems(partly)] == [("drift.ks", "warned")] + assert fd.compare_to_baseline(pd.DataFrame({"price": [5.0] * 100}), base).passed + + +def _verdict(report: fd.DriftReport) -> tuple[bool, list[str]]: + return report.passed, sorted(f.message for f in report.findings if f.status != "passed") + + +def test_saved_baseline_keeps_precision_for_small_values(tmp_path) -> None: + rs = np.random.RandomState(7) + train = pd.DataFrame({"p": rs.normal(2e-6, 4e-7, 2000)}) + today = pd.DataFrame({"p": rs.normal(2e-6, 4e-7, 2000)}) + base = build_baseline(train, name="b") + fd.save_baseline(base, tmp_path / "b.json") + loaded = fd.load_baseline(tmp_path / "b.json") + + col, saved = base.columns["p"], loaded.columns["p"] + assert saved.quantiles == col.quantiles + assert (saved.min, saved.max, saved.mean, saved.std) == (col.min, col.max, col.mean, col.std) + assert _verdict(fd.compare_to_baseline(today, loaded)) == (True, []) + assert _verdict(fd.compare_to_baseline(today, base)) == (True, []) + + +def test_saved_baseline_stats_drop_non_finite_values() -> None: + col = build_baseline(pd.DataFrame({"x": [1.0, 2.0, 3.0]}), name="b").columns["x"] + col.mean, col.std = float("nan"), float("inf") + d = col.to_dict() + assert d["mean"] is None and d["std"] is None and d["max"] == 3.0 diff --git a/tests/test_enterprise_contracts.py b/tests/test_enterprise_contracts.py index b3c12c34..c943f8c8 100644 --- a/tests/test_enterprise_contracts.py +++ b/tests/test_enterprise_contracts.py @@ -2,6 +2,8 @@ from __future__ import annotations +import decimal + import numpy as np import pandas as pd import pytest @@ -11,6 +13,7 @@ from freshdata.enterprise.contracts import ( ColumnContract, DataContract, + _normalize_dtype, build_baseline, compare_to_baseline, load_baseline, @@ -108,6 +111,81 @@ def test_dtype_change_fails(trusted_df): assert any(f.check_id == "schema.dtype_change" for f in report.errors) +@pytest.mark.parametrize( + ("dtype", "family"), + [ + ("double[pyarrow]", "float"), + ("float[pyarrow]", "float"), + ("halffloat[pyarrow]", "float"), + ("decimal128(10, 2)[pyarrow]", "float"), + ("int64[pyarrow]", "int"), + ("uint8[pyarrow]", "int"), + ("bool[pyarrow]", "bool"), + ("string[pyarrow]", "string"), + ("large_string[pyarrow]", "string"), + ("dictionary[pyarrow]", "string"), + ("timestamp[us, tz=UTC][pyarrow]", "datetime"), + ("date32[day][pyarrow]", "datetime"), + # unchanged pandas/numpy names + ("float64", "float"), + ("Int32", "int"), + ("double", "float"), + ("decimal", "float"), + ("category", "string"), + ("datetime64[ns, UTC]", "datetime"), + ], +) +def test_normalize_dtype_families(dtype, family): + assert _normalize_dtype(dtype) == family + + +def _arrow_frame(): + pa = pytest.importorskip("pyarrow") + if not hasattr(pd, "ArrowDtype"): + pytest.skip("pd.ArrowDtype needs pandas >= 1.5") + n = 60 + return pd.DataFrame( + { + "amount": pd.Series(np.linspace(1.0, 60.0, n), dtype=pd.ArrowDtype(pa.float64())), + "price": pd.Series( + [decimal.Decimal(f"{i}.25") for i in range(n)], + dtype=pd.ArrowDtype(pa.decimal128(10, 2)), + ), + "name": pd.Series( + [f"n{i % 5}" for i in range(n)], dtype=pd.ArrowDtype(pa.large_string()) + ), + } + ) + + +def test_arrow_dtypes_satisfy_contract_families(): + df = _arrow_frame() + contract = DataContract( + "c", + ( + ColumnContract("amount", dtype="float"), + ColumnContract("price", dtype="float"), + ColumnContract("name", dtype="string"), + ), + ) + report = fd.enforce_contract(df, contract) + assert report.passed, [f.message for f in report.findings] + + +def test_arrow_numeric_columns_get_numeric_baselines_and_drift(): + df = _arrow_frame() + base = build_baseline(df, name="b") + assert base.columns["amount"].kind == "numeric" + assert base.columns["price"].kind == "numeric" + assert base.columns["name"].kind == "categorical" + assert compare_to_baseline(df, base).passed + + shifted = df.copy() + shifted["amount"] = shifted["amount"] + 1000 + report = compare_to_baseline(shifted, base) + assert {(f.column, f.check_id) for f in report.errors} >= {("amount", "drift.ks")} + + def test_missing_ratio_drift_warns_then_fails(trusted_df): base = build_baseline(trusted_df, name="c") warn_df = trusted_df.copy()