From fdd7df2777b4aa06b7b31ba0c788425dbb0c7be6 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:02:27 +0530 Subject: [PATCH] fix: median imputation crash on nullable integer columns pandas computes a masked-integer median by filling masked slots with iNaT, which numpy 2.5+ refuses to cast into Int8/Int16/Int32/UInt arrays (OverflowError: Python integer ... out of bounds for int8). Every median imputation path crashed on such columns with missing values, including the explicit config of the weekly large performance workflow. Add _util.safe_median, which takes the median of the Float64 view for IntegerArray columns and defers to Series.median() for every other dtype, and use it in steps/missing, engine/missing, simple.fill_missing, MissForest seeding/fallback and seasonal time-series imputation. The numeric_median_fill benchmark baseline now works in float64. --- CHANGELOG.md | 7 ++ benchmarks/performance/baselines.py | 13 ++-- src/freshdata/_util.py | 14 ++++ src/freshdata/engine/missing.py | 8 +-- src/freshdata/imputation/missforest.py | 6 +- src/freshdata/simple.py | 4 +- src/freshdata/steps/missing.py | 3 +- src/freshdata/streaming/_timeseries.py | 3 +- tests/test_nullable_int_median.py | 99 ++++++++++++++++++++++++++ 9 files changed, 142 insertions(+), 15 deletions(-) create mode 100644 tests/test_nullable_int_median.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ae62218..d0b3a14c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,13 @@ adheres to [Semantic Versioning](https://semver.org/). - `explain_clean()` now profiles only post-clean columns that can contribute a decision narrative, avoiding a redundant full-width context pass. +### Fixed +- Median imputation no longer crashes with `OverflowError` on nullable integer + columns (`Int8`/`Int16`/`Int32`/`UInt*`) containing missing values under + numpy 2.5+. Affects `impute="median"`/`"auto"`, the default missing-value + engine, `fill_missing`, MissForest seeding and seasonal time-series + imputation. + ## [2.0.0] - 2026-07-20 Remediation of the July 2026 v1.2.0 production-readiness audit: the unsafe diff --git a/benchmarks/performance/baselines.py b/benchmarks/performance/baselines.py index 59c827be..21015078 100644 --- a/benchmarks/performance/baselines.py +++ b/benchmarks/performance/baselines.py @@ -26,12 +26,15 @@ def _numeric_median_fill(frame: pd.DataFrame) -> pd.DataFrame: + # Work in float64: a fractional median can't go into a nullable integer + # column, and masked-integer median overflows on numpy>=2.5. + numeric = { + column: frame[column].astype("float64") + for column in frame.select_dtypes(include="number").columns + if frame[column].notna().any() + } return frame.assign( - **{ - column: frame[column].fillna(frame[column].median()) - for column in frame.select_dtypes(include="number").columns - if frame[column].notna().any() - } + **{column: values.fillna(values.median()) for column, values in numeric.items()} ) diff --git a/src/freshdata/_util.py b/src/freshdata/_util.py index be3be8d5..ba692d36 100644 --- a/src/freshdata/_util.py +++ b/src/freshdata/_util.py @@ -4,11 +4,25 @@ import hashlib import warnings +from typing import Any import pandas as pd from pandas.errors import PerformanceWarning +def safe_median(s: pd.Series) -> Any: + """``s.median()`` that also works on nullable integer columns. + + pandas computes a masked-integer median by filling masked slots with + ``iNaT``, which overflows narrow dtypes (Int8/16/32, UInt*) on newer numpy. + The ``Float64`` view yields the same value (pandas takes the median in + float64 anyway); every other dtype goes through ``s.median()`` unchanged. + """ + if isinstance(s.array, pd.arrays.IntegerArray): + return s.astype("Float64").median() + return s.median() + + def add_column(df: pd.DataFrame, name: object, values: object) -> None: """Insert a new column in place, suppressing pandas' fragmentation notice. diff --git a/src/freshdata/engine/missing.py b/src/freshdata/engine/missing.py index 8b6ef8a8..a0e83300 100644 --- a/src/freshdata/engine/missing.py +++ b/src/freshdata/engine/missing.py @@ -35,7 +35,7 @@ import pandas as pd from pandas.api.types import is_bool_dtype, is_numeric_dtype -from .._util import add_column +from .._util import add_column, safe_median from ..config import CleanConfig from ..report import CleanReport from ..steps.missing import _mode_value @@ -285,7 +285,7 @@ def _fill_low(df: pd.DataFrame, col: object, ctx: ColumnContext, if ctx.role == "numeric": skewed = ctx.skew is not None and abs(ctx.skew) >= _MEAN_OK_SKEW if skewed or _has_outliers(s): - return _fill(df, col, ctx, report, s.median(), "median", + return _fill(df, col, ctx, report, safe_median(s), "median", rationale="low missingness; skewed or outlier-bearing " "distribution, median is robust", confidence=0.9, model_id=model_id or "median", @@ -336,7 +336,7 @@ def _fill_medium(df: pd.DataFrame, col: object, ctx: ColumnContext, "model-based imputation", confidence=0.75, model_id="knn", min_confidence=min_confidence) - return _fill(df, col, ctx, report, s.median(), "median", + return _fill(df, col, ctx, report, safe_median(s), "median", rationale="medium missingness; median is the safe default " "for numeric columns", confidence=0.8, model_id=model_id or "median", @@ -396,7 +396,7 @@ def _handle_high(df: pd.DataFrame, col: object, ctx: ColumnContext, ) s = df[col] if ctx.role == "numeric": - return _fill(df, col, ctx, report, s.median(), "median", + return _fill(df, col, ctx, report, safe_median(s), "median", rationale=f"high missingness but kept ({keep_reason}); " "conservative median fill", confidence=0.5, risk="high", min_confidence=min_confidence) diff --git a/src/freshdata/imputation/missforest.py b/src/freshdata/imputation/missforest.py index e6781a50..9aae0c1a 100644 --- a/src/freshdata/imputation/missforest.py +++ b/src/freshdata/imputation/missforest.py @@ -15,7 +15,7 @@ import pandas as pd from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_numeric_dtype -from .._util import add_column +from .._util import add_column, safe_median from ..config import CleanConfig from ..engine.context import ColumnContext from ..report import CleanReport @@ -237,7 +237,7 @@ def _initial_filled_frame(self, df: pd.DataFrame) -> pd.DataFrame: dt_value = float(numeric.dropna().median()) if numeric.notna().any() else 0.0 out[col] = numeric.fillna(dt_value) elif is_numeric_dtype(s) and not is_bool_dtype(s): - numeric_value = s.median() if s.notna().any() else 0.0 + numeric_value = safe_median(s) if s.notna().any() else 0.0 out[col] = s.astype("float64").fillna(numeric_value) elif is_bool_dtype(s): bool_value = _mode_value(s) @@ -346,7 +346,7 @@ def _fallback_fill( reason: str, ) -> None: s = df[col] - value = s.median() if ctx.role == "numeric" and s.notna().any() else _mode_value(s) + value = safe_median(s) if ctx.role == "numeric" and s.notna().any() else _mode_value(s) if value is None or pd.isna(value): value = "Missing" if ctx.role in ("categorical", "boolean") else None if value is None or pd.isna(value): diff --git a/src/freshdata/simple.py b/src/freshdata/simple.py index 5c47f854..8374374e 100644 --- a/src/freshdata/simple.py +++ b/src/freshdata/simple.py @@ -21,6 +21,8 @@ import pandas as pd from pandas.api.types import is_bool_dtype, is_numeric_dtype +from ._util import safe_median + __all__ = [ "detect_outliers", "fill_missing", @@ -153,7 +155,7 @@ def fill_missing( if strategy == "mode": fill = _mode_value(s) elif is_numeric_dtype(s): - fill = s.mean() if strategy == "mean" else s.median() + fill = s.mean() if strategy == "mean" else safe_median(s) else: fill = None # mean/median are undefined for non-numeric columns if fill is None: diff --git a/src/freshdata/steps/missing.py b/src/freshdata/steps/missing.py index 38426b74..976ffb46 100644 --- a/src/freshdata/steps/missing.py +++ b/src/freshdata/steps/missing.py @@ -12,6 +12,7 @@ import pandas as pd from pandas.api.types import is_bool_dtype, is_numeric_dtype +from .._util import safe_median from ..config import CleanConfig from ..report import CleanReport @@ -35,7 +36,7 @@ def _fill_value(s: pd.Series, strategy: str) -> Any | None: if strategy in ("mean", "median"): if not numeric: return None # not defined for this dtype; caller reports the skip - return s.mean() if strategy == "mean" else s.median() + return s.mean() if strategy == "mean" else safe_median(s) return _mode_value(s) diff --git a/src/freshdata/streaming/_timeseries.py b/src/freshdata/streaming/_timeseries.py index 22924e77..ae1953b2 100644 --- a/src/freshdata/streaming/_timeseries.py +++ b/src/freshdata/streaming/_timeseries.py @@ -24,6 +24,7 @@ import numpy as np import pandas as pd +from .._util import safe_median from ..config import CleanConfig from ..engine.context import infer_role from ..report import CleanReport @@ -460,7 +461,7 @@ def fill_group(g: pd.DataFrame, col: str = col) -> pd.DataFrame: missing = s.isna() if not missing.any(): return g - global_med = s.median() + global_med = safe_median(s) if season_fn is not None: keys = pd.Series(season_fn(pd.DatetimeIndex(g[ts_col])), index=g.index) season_med = s.groupby(keys).transform("median") diff --git a/tests/test_nullable_int_median.py b/tests/test_nullable_int_median.py new file mode 100644 index 00000000..e7a9ca3d --- /dev/null +++ b/tests/test_nullable_int_median.py @@ -0,0 +1,99 @@ +"""Median imputation on nullable integer columns. + +pandas computes a masked-integer median by filling masked slots with ``iNaT``; +on newer numpy that overflows Int8/Int16/Int32 and unsigned dtypes +(``OverflowError: Python integer ... out of bounds for int8``). Every median +imputation path must survive those columns and keep Int64/float behaviour. +""" + +import numpy as np +import pandas as pd +import pytest + +import freshdata as fd +from freshdata._util import safe_median +from freshdata.engine import missing, model_select +from freshdata.engine.utils import _has_outliers + +NARROW = ["Int8", "Int16", "Int32", "UInt8", "UInt64"] +KEEP_ROWS = {"drop_empty_rows": False, "drop_duplicates": False, "verbose": False} + + +def _series(dtype: str, n: int = 40) -> pd.Series: + # 37 non-missing values -> the median is an observed (integer) value. + s = pd.Series(pd.array([i % 7 for i in range(n)], dtype=dtype)) + s.iloc[[3, 11, 25]] = pd.NA + return s + + +def _expected_median(s: pd.Series) -> float: + return float(np.median(s.dropna().to_numpy(dtype="float64"))) + + +@pytest.mark.parametrize("dtype", [*NARROW, "Int64", "Float32", "float64"]) +def test_safe_median_matches_float_median(dtype): + s = _series(dtype) + assert safe_median(s) == _expected_median(s) + + +def test_safe_median_leaves_non_integer_dtypes_untouched(): + s = _series("Float32") + value = safe_median(s) + assert type(value) is type(s.median()) + assert value == s.median() + + +@pytest.mark.parametrize("strategy", ["median", "auto"]) +@pytest.mark.parametrize("dtype", [*NARROW, "Int64"]) +def test_explicit_impute_median_on_nullable_int(dtype, strategy): + s = _series(dtype) + df = pd.DataFrame({"v": s}) + out, report = fd.clean( + df, impute=strategy, strategy="conservative", return_report=True, **KEEP_ROWS + ) + assert out["v"].isna().sum() == 0 + assert out["v"].iloc[3] == _expected_median(s) + assert any(a.step == "impute" and strategy in a.description for a in report) + + +@pytest.mark.parametrize("dtype", [*NARROW, "Int64"]) +def test_fill_missing_median_on_nullable_int(dtype): + s = _series(dtype) + out = fd.fill_missing(pd.DataFrame({"v": s}), method="median") + assert out["v"].isna().sum() == 0 + assert out["v"].iloc[3] == _expected_median(s) + + +@pytest.mark.parametrize("dtype", NARROW) +def test_default_engine_survives_nullable_int_with_outliers(dtype): + rng = np.random.default_rng(0) + values = rng.integers(10, 60, 100) + values[:4] = 120 # outliers -> the engine's low band picks median + s = pd.Series(pd.array(values, dtype=dtype)) + s.iloc[[10, 20, 30]] = pd.NA + df = pd.DataFrame({"v": s, "x": rng.normal(0, 1, 100)}) + out, report = fd.clean(df, return_report=True, **KEEP_ROWS) + filled = [a for a in report if a.step == "missing" and a.column == "v"] + assert filled, "the engine must record a decision for the column" + if "median" in filled[0].description: + assert out["v"].isna().sum() == 0 + + +@pytest.mark.parametrize("dtype", ["Int16", "UInt8"]) +def test_seasonal_imputation_global_median_on_nullable_int(dtype): + idx = pd.date_range("2024-01-01", periods=96, freq="h") + s = pd.Series(pd.array([10 + (i % 5) for i in range(96)], dtype=dtype)) + s.iloc[[5, 40, 72]] = pd.NA + df = pd.DataFrame({"t": idx, "v": s}) + out = fd.clean_timeseries( + df, timestamp_column="t", max_interpolation_gap=0, + seasonal_period="hour", seasonal_imputation_enabled=True, + ) + assert out["v"].isna().sum() == 0 + + +def test_has_outliers_single_definition(): + assert missing._has_outliers is _has_outliers + assert model_select._has_outliers is _has_outliers + assert _has_outliers(pd.Series([1.0, 2.0, 3.0, 4.0, 100.0])) + assert not _has_outliers(pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]))