From e90ec433fb0fbf54f92825b3f083e72bdeb40e64 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:19:09 +0530 Subject: [PATCH] fix: nullable integer streaming stats on pandas < 2 test-matrix (3.9) has failed on main since #187: with pandas 1.5, the streaming state converted numeric columns via pd.to_numeric(s).to_numpy(dtype="float64"), which raises "cannot convert to 'float64'-dtype NumPy array with missing values" for nullable (masked) integer columns with NA. fd.clean_timeseries and StreamingCleaner hit it on any Int*/UInt* column with gaps. Pass na_value=np.nan there and in the time-series interpolation helper, which has the same conversion. Also pin the alert step #188 added to fetch-fixtures.yml, which #190's SHA sweep predated. --- .github/workflows/fetch-fixtures.yml | 2 +- src/freshdata/streaming/_state.py | 4 +++- src/freshdata/streaming/_timeseries.py | 13 +++++++------ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/fetch-fixtures.yml b/.github/workflows/fetch-fixtures.yml index a69dd13a..851dd269 100644 --- a/.github/workflows/fetch-fixtures.yml +++ b/.github/workflows/fetch-fixtures.yml @@ -33,7 +33,7 @@ jobs: run: pytest tests/test_online_datasets.py -m "not online" -q --no-cov - name: Open/refresh alert issue on failure if: failure() && github.event_name == 'schedule' - uses: actions/github-script@v7 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | const title = "weekly online fixture refresh failing"; diff --git a/src/freshdata/streaming/_state.py b/src/freshdata/streaming/_state.py index e8faec9e..88e9add7 100644 --- a/src/freshdata/streaming/_state.py +++ b/src/freshdata/streaming/_state.py @@ -107,7 +107,9 @@ def update(self, s: pd.Series) -> None: self.nunique_stat.update(np.array([float(s.nunique(dropna=True))])) if is_numeric_dtype(s) and not is_bool_dtype(s): - values = pd.to_numeric(s, errors="coerce").to_numpy(dtype="float64") + # na_value: nullable (masked) columns with missing cells refuse a + # plain float64 conversion on pandas < 2. + values = pd.to_numeric(s, errors="coerce").to_numpy(dtype="float64", na_value=np.nan) values = values[~np.isnan(values)] if values.size: self._welford.update(values) diff --git a/src/freshdata/streaming/_timeseries.py b/src/freshdata/streaming/_timeseries.py index ae1953b2..787a4d00 100644 --- a/src/freshdata/streaming/_timeseries.py +++ b/src/freshdata/streaming/_timeseries.py @@ -428,18 +428,19 @@ def _interp_series(s: pd.Series, ts: pd.Series, method: str) -> pd.Series: return s.ffill() if method == "bfill": return s.bfill() + # na_value: nullable (masked) columns with missing cells refuse a plain + # float64 conversion on pandas < 2. + values = s.to_numpy(dtype="float64", na_value=np.nan) if method == "time": - tmp = pd.Series(s.to_numpy(dtype="float64"), - index=pd.DatetimeIndex(ts)) + tmp = pd.Series(values, index=pd.DatetimeIndex(ts)) try: out = tmp.interpolate(method="time", limit_area="inside") except ValueError: # non-monotonic / non-datetime index → linear fallback - out = pd.Series(s.to_numpy(dtype="float64")).interpolate( - method="linear", limit_area="inside") + out = pd.Series(values).interpolate(method="linear", limit_area="inside") return pd.Series(out.to_numpy(), index=s.index) # linear - return pd.Series(s.to_numpy(dtype="float64"), - index=s.index).interpolate(method="linear", limit_area="inside") + return pd.Series(values, index=s.index).interpolate( + method="linear", limit_area="inside") # -- step 3: seasonal imputation -------------------------------------------