Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@ adheres to [Semantic Versioning](https://semver.org/).
- `explain_clean()` now profiles only post-clean columns that can contribute a
decision narrative, avoiding a redundant full-width context pass.

### Fixed
- Median imputation no longer crashes with `OverflowError` on nullable integer
columns (`Int8`/`Int16`/`Int32`/`UInt*`) containing missing values under
numpy 2.5+. Affects `impute="median"`/`"auto"`, the default missing-value
engine, `fill_missing`, MissForest seeding and seasonal time-series
imputation.

## [2.0.0] - 2026-07-20

Remediation of the July 2026 v1.2.0 production-readiness audit: the unsafe
Expand Down
13 changes: 8 additions & 5 deletions benchmarks/performance/baselines.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,15 @@


def _numeric_median_fill(frame: pd.DataFrame) -> pd.DataFrame:
# Work in float64: a fractional median can't go into a nullable integer
# column, and masked-integer median overflows on numpy>=2.5.
numeric = {
column: frame[column].astype("float64")
for column in frame.select_dtypes(include="number").columns
if frame[column].notna().any()
}
return frame.assign(
**{
column: frame[column].fillna(frame[column].median())
for column in frame.select_dtypes(include="number").columns
if frame[column].notna().any()
}
**{column: values.fillna(values.median()) for column, values in numeric.items()}
)


Expand Down
14 changes: 14 additions & 0 deletions src/freshdata/_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,25 @@

import hashlib
import warnings
from typing import Any

import pandas as pd
from pandas.errors import PerformanceWarning


def safe_median(s: pd.Series) -> Any:
"""``s.median()`` that also works on nullable integer columns.

pandas computes a masked-integer median by filling masked slots with
``iNaT``, which overflows narrow dtypes (Int8/16/32, UInt*) on newer numpy.
The ``Float64`` view yields the same value (pandas takes the median in
float64 anyway); every other dtype goes through ``s.median()`` unchanged.
"""
if isinstance(s.array, pd.arrays.IntegerArray):
return s.astype("Float64").median()
return s.median()


def add_column(df: pd.DataFrame, name: object, values: object) -> None:
"""Insert a new column in place, suppressing pandas' fragmentation notice.

Expand Down
8 changes: 4 additions & 4 deletions src/freshdata/engine/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import pandas as pd
from pandas.api.types import is_bool_dtype, is_numeric_dtype

from .._util import add_column
from .._util import add_column, safe_median
from ..config import CleanConfig
from ..report import CleanReport
from ..steps.missing import _mode_value
Expand Down Expand Up @@ -285,7 +285,7 @@ def _fill_low(df: pd.DataFrame, col: object, ctx: ColumnContext,
if ctx.role == "numeric":
skewed = ctx.skew is not None and abs(ctx.skew) >= _MEAN_OK_SKEW
if skewed or _has_outliers(s):
return _fill(df, col, ctx, report, s.median(), "median",
return _fill(df, col, ctx, report, safe_median(s), "median",
rationale="low missingness; skewed or outlier-bearing "
"distribution, median is robust",
confidence=0.9, model_id=model_id or "median",
Expand Down Expand Up @@ -336,7 +336,7 @@ def _fill_medium(df: pd.DataFrame, col: object, ctx: ColumnContext,
"model-based imputation",
confidence=0.75, model_id="knn",
min_confidence=min_confidence)
return _fill(df, col, ctx, report, s.median(), "median",
return _fill(df, col, ctx, report, safe_median(s), "median",
rationale="medium missingness; median is the safe default "
"for numeric columns",
confidence=0.8, model_id=model_id or "median",
Expand Down Expand Up @@ -396,7 +396,7 @@ def _handle_high(df: pd.DataFrame, col: object, ctx: ColumnContext,
)
s = df[col]
if ctx.role == "numeric":
return _fill(df, col, ctx, report, s.median(), "median",
return _fill(df, col, ctx, report, safe_median(s), "median",
rationale=f"high missingness but kept ({keep_reason}); "
"conservative median fill",
confidence=0.5, risk="high", min_confidence=min_confidence)
Expand Down
6 changes: 3 additions & 3 deletions src/freshdata/imputation/missforest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import pandas as pd
from pandas.api.types import is_bool_dtype, is_datetime64_any_dtype, is_numeric_dtype

from .._util import add_column
from .._util import add_column, safe_median
from ..config import CleanConfig
from ..engine.context import ColumnContext
from ..report import CleanReport
Expand Down Expand Up @@ -237,7 +237,7 @@ def _initial_filled_frame(self, df: pd.DataFrame) -> pd.DataFrame:
dt_value = float(numeric.dropna().median()) if numeric.notna().any() else 0.0
out[col] = numeric.fillna(dt_value)
elif is_numeric_dtype(s) and not is_bool_dtype(s):
numeric_value = s.median() if s.notna().any() else 0.0
numeric_value = safe_median(s) if s.notna().any() else 0.0
out[col] = s.astype("float64").fillna(numeric_value)
elif is_bool_dtype(s):
bool_value = _mode_value(s)
Expand Down Expand Up @@ -346,7 +346,7 @@ def _fallback_fill(
reason: str,
) -> None:
s = df[col]
value = s.median() if ctx.role == "numeric" and s.notna().any() else _mode_value(s)
value = safe_median(s) if ctx.role == "numeric" and s.notna().any() else _mode_value(s)
if value is None or pd.isna(value):
value = "Missing" if ctx.role in ("categorical", "boolean") else None
if value is None or pd.isna(value):
Expand Down
4 changes: 3 additions & 1 deletion src/freshdata/simple.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import pandas as pd
from pandas.api.types import is_bool_dtype, is_numeric_dtype

from ._util import safe_median

__all__ = [
"detect_outliers",
"fill_missing",
Expand Down Expand Up @@ -153,7 +155,7 @@ def fill_missing(
if strategy == "mode":
fill = _mode_value(s)
elif is_numeric_dtype(s):
fill = s.mean() if strategy == "mean" else s.median()
fill = s.mean() if strategy == "mean" else safe_median(s)
else:
fill = None # mean/median are undefined for non-numeric columns
if fill is None:
Expand Down
3 changes: 2 additions & 1 deletion src/freshdata/steps/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import pandas as pd
from pandas.api.types import is_bool_dtype, is_numeric_dtype

from .._util import safe_median
from ..config import CleanConfig
from ..report import CleanReport

Expand All @@ -35,7 +36,7 @@ def _fill_value(s: pd.Series, strategy: str) -> Any | None:
if strategy in ("mean", "median"):
if not numeric:
return None # not defined for this dtype; caller reports the skip
return s.mean() if strategy == "mean" else s.median()
return s.mean() if strategy == "mean" else safe_median(s)
return _mode_value(s)


Expand Down
3 changes: 2 additions & 1 deletion src/freshdata/streaming/_timeseries.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import numpy as np
import pandas as pd

from .._util import safe_median
from ..config import CleanConfig
from ..engine.context import infer_role
from ..report import CleanReport
Expand Down Expand Up @@ -460,7 +461,7 @@ def fill_group(g: pd.DataFrame, col: str = col) -> pd.DataFrame:
missing = s.isna()
if not missing.any():
return g
global_med = s.median()
global_med = safe_median(s)
if season_fn is not None:
keys = pd.Series(season_fn(pd.DatetimeIndex(g[ts_col])), index=g.index)
season_med = s.groupby(keys).transform("median")
Expand Down
99 changes: 99 additions & 0 deletions tests/test_nullable_int_median.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
"""Median imputation on nullable integer columns.

pandas computes a masked-integer median by filling masked slots with ``iNaT``;
on newer numpy that overflows Int8/Int16/Int32 and unsigned dtypes
(``OverflowError: Python integer ... out of bounds for int8``). Every median
imputation path must survive those columns and keep Int64/float behaviour.
"""

import numpy as np
import pandas as pd
import pytest

import freshdata as fd
from freshdata._util import safe_median
from freshdata.engine import missing, model_select
from freshdata.engine.utils import _has_outliers

NARROW = ["Int8", "Int16", "Int32", "UInt8", "UInt64"]
KEEP_ROWS = {"drop_empty_rows": False, "drop_duplicates": False, "verbose": False}


def _series(dtype: str, n: int = 40) -> pd.Series:
# 37 non-missing values -> the median is an observed (integer) value.
s = pd.Series(pd.array([i % 7 for i in range(n)], dtype=dtype))
s.iloc[[3, 11, 25]] = pd.NA
return s


def _expected_median(s: pd.Series) -> float:
return float(np.median(s.dropna().to_numpy(dtype="float64")))


@pytest.mark.parametrize("dtype", [*NARROW, "Int64", "Float32", "float64"])
def test_safe_median_matches_float_median(dtype):
s = _series(dtype)
assert safe_median(s) == _expected_median(s)


def test_safe_median_leaves_non_integer_dtypes_untouched():
s = _series("Float32")
value = safe_median(s)
assert type(value) is type(s.median())
assert value == s.median()


@pytest.mark.parametrize("strategy", ["median", "auto"])
@pytest.mark.parametrize("dtype", [*NARROW, "Int64"])
def test_explicit_impute_median_on_nullable_int(dtype, strategy):
s = _series(dtype)
df = pd.DataFrame({"v": s})
out, report = fd.clean(
df, impute=strategy, strategy="conservative", return_report=True, **KEEP_ROWS
)
assert out["v"].isna().sum() == 0
assert out["v"].iloc[3] == _expected_median(s)
assert any(a.step == "impute" and strategy in a.description for a in report)


@pytest.mark.parametrize("dtype", [*NARROW, "Int64"])
def test_fill_missing_median_on_nullable_int(dtype):
s = _series(dtype)
out = fd.fill_missing(pd.DataFrame({"v": s}), method="median")
assert out["v"].isna().sum() == 0
assert out["v"].iloc[3] == _expected_median(s)


@pytest.mark.parametrize("dtype", NARROW)
def test_default_engine_survives_nullable_int_with_outliers(dtype):
rng = np.random.default_rng(0)
values = rng.integers(10, 60, 100)
values[:4] = 120 # outliers -> the engine's low band picks median
s = pd.Series(pd.array(values, dtype=dtype))
s.iloc[[10, 20, 30]] = pd.NA
df = pd.DataFrame({"v": s, "x": rng.normal(0, 1, 100)})
out, report = fd.clean(df, return_report=True, **KEEP_ROWS)
filled = [a for a in report if a.step == "missing" and a.column == "v"]
assert filled, "the engine must record a decision for the column"
if "median" in filled[0].description:
assert out["v"].isna().sum() == 0


@pytest.mark.parametrize("dtype", ["Int16", "UInt8"])
def test_seasonal_imputation_global_median_on_nullable_int(dtype):
idx = pd.date_range("2024-01-01", periods=96, freq="h")
s = pd.Series(pd.array([10 + (i % 5) for i in range(96)], dtype=dtype))
s.iloc[[5, 40, 72]] = pd.NA
df = pd.DataFrame({"t": idx, "v": s})
out = fd.clean_timeseries(
df, timestamp_column="t", max_interpolation_gap=0,
seasonal_period="hour", seasonal_imputation_enabled=True,
)
assert out["v"].isna().sum() == 0


def test_has_outliers_single_definition():
assert missing._has_outliers is _has_outliers
assert model_select._has_outliers is _has_outliers
assert _has_outliers(pd.Series([1.0, 2.0, 3.0, 4.0, 100.0]))
assert not _has_outliers(pd.Series([1.0, 2.0, 3.0, 4.0, 5.0]))
Loading