Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 101 additions & 39 deletions src/freshdata/enterprise/contracts.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,21 @@ def _round(value: float | None, ndigits: int = 6) -> float | None:
return round(float(value), ndigits)


def _full_precision(value: float | None) -> float | None:
"""*value* as an unrounded float (JSON round-trips it exactly); ``None`` if NaN/inf.

Baseline statistics are stored this way: rounding to decimal places erases
small-magnitude columns, and any rounding makes stored quantiles differ from
the data values they came from, which the drift checks compare exactly.
"""
if value is None:
return None
v = float(value)
if math.isnan(v) or math.isinf(v):
return None
return v


def _hash_label(value: str) -> str:
"""Stable, non-reversible category label for PII-safe baselines."""
return "h:" + hashlib.sha1(value.encode("utf-8")).hexdigest()[:16]
Expand All @@ -99,19 +114,26 @@ def _normalize_dtype(dtype: str | None) -> str | None:
``int64``/``Int32`` → ``int``; ``float64`` → ``float``; ``object``/``string``
→ ``string``; ``datetime64[ns, UTC]`` → ``datetime``; ``bool`` → ``bool``.
Keeps drift comparison robust to width and nullable-extension variants.
``pd.ArrowDtype`` names (``double[pyarrow]``, ``large_string[pyarrow]``,
``decimal128(10, 2)[pyarrow]``, ``timestamp[us, tz=UTC][pyarrow]``, …) map
to the same families.
"""
if dtype is None:
return None
d = dtype.strip().lower()
if d.endswith("[pyarrow]"):
d = d[: -len("[pyarrow]")]
if d.startswith(("int", "uint")) or d in ("int", "integer"):
return "int"
if d.startswith("float") or d in ("double", "decimal"):
if d.startswith(("float", "halffloat", "decimal")) or d == "double":
return "float"
if d.startswith("bool"):
return "bool"
if d.startswith("dictionary"): # Arrow's categorical encoding
return "string"
if d.startswith(("datetime", "timestamp")) or "date" in d:
return "datetime"
if d.startswith(("object", "string", "str", "category", "utf8")):
if d.startswith(("object", "string", "str", "category", "utf8", "large_string", "large_utf8")):
return "string"
return d

Expand Down Expand Up @@ -316,11 +338,11 @@ def to_dict(self) -> dict[str, Any]:
"n_unique": self.n_unique,
"n_rows": self.n_rows,
"sample_values": list(self.sample_values),
"min": _round(self.min),
"max": _round(self.max),
"mean": _round(self.mean),
"std": _round(self.std),
"quantiles": {k: _round(v) for k, v in self.quantiles.items()},
"min": _full_precision(self.min),
"max": _full_precision(self.max),
"mean": _full_precision(self.mean),
"std": _full_precision(self.std),
"quantiles": {k: _full_precision(v) for k, v in self.quantiles.items()},
"top_values": list(self.top_values),
"frequencies": {k: _round(v) for k, v in self.frequencies.items()},
"min_timestamp": self.min_timestamp,
Expand Down Expand Up @@ -702,7 +724,8 @@ def _profile_column(series: pd.Series, *, n_rows: int, include_samples: bool) ->
cb.sample_values = tuple(str(v) for v in uniques)

if family in ("int", "float") and len(non_null):
numeric = pd.to_numeric(non_null, errors="coerce").dropna()
# float: Arrow decimals come back as Decimal objects, which can't take std().
numeric = pd.to_numeric(non_null, errors="coerce").dropna().astype(float)
if len(numeric):
cb.min = float(numeric.min())
cb.max = float(numeric.max())
Expand Down Expand Up @@ -803,20 +826,66 @@ def _ks_statistic(cb: ColumnBaseline, current: pd.Series) -> float | None:
n = len(vals)
if n == 0:
return None
knots, p_lo, p_hi = _cdf_knots(pts)
cur = np.sort(vals)

# At a stored value x the quantiles only bound the baseline CDF:
# F(x-) <= p_lo and F(x) >= p_hi. The jump F(x) - F(x-) is a point mass,
# which can be no larger than the baseline's distinct-value count allows.
# Measure the distance to the nearest baseline CDF within those bounds, so
# ties (discrete, zero-inflated or constant columns) don't read as drift.
mass = _max_point_mass(cb)
below = np.searchsorted(cur, knots, side="left") / n
at = np.searchsorted(cur, knots, side="right") / n
at_knots = np.maximum.reduce(
[
below - p_lo,
np.minimum(p_hi - mass, p_lo) - below,
p_hi - at,
at - np.maximum(p_lo + mass, p_hi),
]
)
stat = max(0.0, float(np.max(at_knots)))

# Between knots the baseline CDF is interpolated linearly, as before.
grid = np.quantile(vals, [p for p, _ in pts])
grid = grid[~np.isin(grid, knots)]
if len(grid):
f_base = _cdf_between_knots(grid, knots, p_lo, p_hi)
f_cur = np.searchsorted(cur, grid, side="right") / n
stat = max(stat, float(np.max(np.abs(f_base - f_cur))))
return stat


def _cdf_knots(pts: list[tuple[float, float]]) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Distinct knot values with the lowest and highest probability stored at each."""
xs = np.array([v for _, v in pts], dtype=float)
ps = np.array([p for p, _ in pts], dtype=float)
# Ensure strictly increasing xs for interpolation.
order = np.argsort(xs, kind="stable")
xs, ps = xs[order], ps[order]
keep = np.concatenate(([True], np.diff(xs) > 0))
xs, ps = xs[keep], ps[keep]
if len(xs) < 2:
return None

grid = np.unique(np.concatenate([xs, np.quantile(vals, ps)]))
f_base = np.interp(grid, xs, ps, left=0.0, right=1.0)
f_cur = np.searchsorted(np.sort(vals), grid, side="right") / n
return float(np.max(np.abs(f_base - f_cur)))
knots, first = np.unique(xs, return_index=True)
return knots, np.minimum.reduceat(ps, first), np.maximum.reduceat(ps, first)


def _max_point_mass(cb: ColumnBaseline) -> float:
"""Largest share of non-null baseline rows a single value can hold."""
n = cb.n_rows * (1.0 - (cb.missing_ratio or 0.0))
if n < 1 or cb.n_unique < 1:
return 1.0
return min(1.0, max(0.0, (n - cb.n_unique + 1) / n))


def _cdf_between_knots(
x: np.ndarray, knots: np.ndarray, p_lo: np.ndarray, p_hi: np.ndarray
) -> np.ndarray:
"""Baseline CDF at points *x* that are not knots: linear between, 0/1 outside."""
idx = np.searchsorted(knots, x, side="right") - 1
out = np.where(idx < 0, 0.0, 1.0)
inside = (idx >= 0) & (idx < len(knots) - 1)
i = idx[inside]
frac = (x[inside] - knots[i]) / (knots[i + 1] - knots[i])
out[inside] = p_hi[i] + frac * (p_lo[i + 1] - p_hi[i])
return out


def _psi_numeric(cb: ColumnBaseline, current: pd.Series) -> float | None:
Expand All @@ -828,28 +897,21 @@ def _psi_numeric(cb: ColumnBaseline, current: pd.Series) -> float | None:
n = len(vals)
if n == 0:
return None
probs = [p for p, _ in pts]
edges = [v for _, v in pts]
# Merge bins whose edges collapse (repeated quantile values).
merged_edges: list[float] = [edges[0]]
merged_expected: list[float] = []
acc = 0.0
for i in range(1, len(edges)):
acc += probs[i] - probs[i - 1]
if edges[i] > merged_edges[-1]:
merged_edges.append(edges[i])
merged_expected.append(acc)
acc = 0.0
if acc > 0 and merged_expected:
merged_expected[-1] += acc
if len(merged_edges) < 2:
knots, p_lo, _ = _cdf_knots(pts)
if len(knots) < 2:
return None
inner = np.array(merged_edges[1:-1], dtype=float)
idx = np.searchsorted(inner, vals, side="right") if len(inner) else np.zeros(n, dtype=int)
actual_counts = np.bincount(idx, minlength=len(merged_expected)).astype(float)
actual = actual_counts / n
expected = np.array(merged_expected, dtype=float)
expected = expected / expected.sum() if expected.sum() else expected
# Bins run from one distinct quantile value to the next; repeated values
# collapse into one edge. The expected share below an edge is the lowest
# probability stored at it. Current values equal to an edge are split
# across it the same way, so a point mass on a quantile isn't counted
# wholly in the upper bin.
inner = knots[1:-1]
cut = p_lo[1:-1]
cur = np.sort(vals)
below = np.searchsorted(cur, inner, side="left") / n
at = np.searchsorted(cur, inner, side="right") / n
expected = np.diff(np.concatenate(([0.0], cut, [1.0])))
actual = np.diff(np.concatenate(([0.0], np.clip(cut, below, at), [1.0])))
return _psi(expected, actual)


Expand Down
108 changes: 108 additions & 0 deletions tests/test_baseline_compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@

from __future__ import annotations

import numpy as np
import pandas as pd
import pytest

import freshdata as fd
from freshdata import build_baseline
Expand Down Expand Up @@ -66,3 +68,109 @@ def test_prebuilt_baseline_still_works() -> None:
bl = build_baseline(base, name="weekly")
diff = fd.compare_to_baseline(base, bl) # positional, original API
assert diff.passed


# ---------------------------------------------------------------------------
# Distribution drift maths: ties, constant columns and saved precision
# (#234, #235, #275)
# ---------------------------------------------------------------------------


def _drift_problems(report: fd.DriftReport) -> list[tuple[str | None, str, str]]:
return [
(f.check_id, f.status, f.message)
for f in report.findings
if (f.check_id or "").startswith("drift.") and f.status != "passed"
]


_rs = np.random.RandomState(3)
_TIED_COLUMNS = {
"zero_inflated_counts": [0] * 99 + [1],
"small_int_ratings": [1, 2, 3] * 40,
"constant": [5.0] * 100,
# The median lands on a value that holds most of the rows between p25 and p75.
"atom_straddling_quantiles": [0] * 9 + [1] * 15 + [2] * 8,
"zero_inflated_continuous": list(np.where(_rs.rand(500) < 0.6, 0.0, _rs.exponential(5, 500))),
"thirds": [k / 3 for k in (0, 1, 2)] * 40,
"inexact_tenths": [0.1 * k for k in (1, 2, 3)] * 40, # 0.30000000000000004
}


def test_issue_234_repro_frames_match_their_own_baseline() -> None:
df = pd.DataFrame({"refunds": [0] * 99 + [1]})
rep = fd.compare_to_baseline(df, fd.build_baseline(df, name="self"))
df2 = pd.DataFrame({"rating": [1, 2, 3] * 40})
rep2 = fd.compare_to_baseline(df2, fd.build_baseline(df2, name="self"))
assert rep.passed and rep2.passed
assert _drift_problems(rep) == [] and _drift_problems(rep2) == []


@pytest.mark.parametrize("values", list(_TIED_COLUMNS.values()), ids=list(_TIED_COLUMNS))
def test_frame_with_tied_values_matches_its_own_baseline(values: list[float], tmp_path) -> None:
df = pd.DataFrame({"x": values})
base = build_baseline(df, name="self")
fd.save_baseline(base, tmp_path / "self.json")
for bl in (base, fd.load_baseline(tmp_path / "self.json")):
report = fd.compare_to_baseline(df, bl)
assert report.passed
assert _drift_problems(report) == []


def test_tied_baseline_still_flags_real_drift() -> None:
refunds = build_baseline(pd.DataFrame({"refunds": [0] * 99 + [1]}), name="b")
more_refunds = pd.DataFrame({"refunds": [0] * 80 + [1] * 20})
assert not fd.compare_to_baseline(more_refunds, refunds).passed
ratings = build_baseline(pd.DataFrame({"rating": [1, 2, 3] * 40}), name="b")
assert not fd.compare_to_baseline(pd.DataFrame({"rating": [3] * 120}), ratings).passed


def test_point_mass_on_a_continuous_baseline_quantile_is_still_drift() -> None:
# e.g. missing values imputed with the median: the baseline had no such spike.
rs = np.random.RandomState(0)
values = rs.normal(0, 1, 2000)
current = values.copy()
current[:1200] = np.median(values)
base = build_baseline(pd.DataFrame({"x": values}), name="b")
report = fd.compare_to_baseline(pd.DataFrame({"x": current}), base)
assert ("drift.ks", "failed") in {(f.check_id, f.status) for f in report.findings}


def test_constant_baseline_column_still_checks_distribution() -> None:
base = build_baseline(pd.DataFrame({"price": [5.0] * 100}), name="b")

shifted = fd.compare_to_baseline(pd.DataFrame({"price": [1000.0] * 100}), base)
assert not shifted.passed
ks = [f for f in shifted.findings if f.check_id == "drift.ks"]
assert [(f.status, f.current_value) for f in ks] == [("failed", 1.0)]
assert shifted.distribution_drift["price"]["ks"] == 1.0

partly = fd.compare_to_baseline(pd.DataFrame({"price": [4.0] * 20 + [5.0] * 80}), base)
assert [(f[0], f[1]) for f in _drift_problems(partly)] == [("drift.ks", "warned")]
assert fd.compare_to_baseline(pd.DataFrame({"price": [5.0] * 100}), base).passed


def _verdict(report: fd.DriftReport) -> tuple[bool, list[str]]:
return report.passed, sorted(f.message for f in report.findings if f.status != "passed")


def test_saved_baseline_keeps_precision_for_small_values(tmp_path) -> None:
rs = np.random.RandomState(7)
train = pd.DataFrame({"p": rs.normal(2e-6, 4e-7, 2000)})
today = pd.DataFrame({"p": rs.normal(2e-6, 4e-7, 2000)})
base = build_baseline(train, name="b")
fd.save_baseline(base, tmp_path / "b.json")
loaded = fd.load_baseline(tmp_path / "b.json")

col, saved = base.columns["p"], loaded.columns["p"]
assert saved.quantiles == col.quantiles
assert (saved.min, saved.max, saved.mean, saved.std) == (col.min, col.max, col.mean, col.std)
assert _verdict(fd.compare_to_baseline(today, loaded)) == (True, [])
assert _verdict(fd.compare_to_baseline(today, base)) == (True, [])


def test_saved_baseline_stats_drop_non_finite_values() -> None:
col = build_baseline(pd.DataFrame({"x": [1.0, 2.0, 3.0]}), name="b").columns["x"]
col.mean, col.std = float("nan"), float("inf")
d = col.to_dict()
assert d["mean"] is None and d["std"] is None and d["max"] == 3.0
Loading
Loading