From d10a1cf0e8f9e05a2072b66d16d9b78470af0d4e Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 14 Sep 2026 23:15:59 +0530 Subject: [PATCH] fix: keep DuckDB and Polars results out of pandas unless requested DuckDB (#52): output_format="arrow" / "polars" fetched every result with fetchdf() and then converted the pandas frame again. Fetch directly with to_arrow_table() (fetch_arrow_table() on older duckdb) or .pl() instead, keeping fetchdf() for pandas output and when the native semantic stage, which does not accept Arrow tables, runs afterwards. _frame_stats learns pyarrow tables and _convert_output passes them through. Polars (#53): duplicate detection and dedup counted rows with plain .collect() calls that re-evaluate the whole upstream plan in memory. Route them through the streaming collect path via a new _count helper; _collect now accepts engine_config=None (streaming default). --- CHANGELOG.md | 7 + src/freshdata/execution/__init__.py | 6 +- src/freshdata/execution/_report.py | 10 ++ src/freshdata/execution/backends/_duckdb.py | 15 +- src/freshdata/execution/backends/_polars.py | 17 ++- .../test_execution/test_out_of_core_fetch.py | 128 ++++++++++++++++++ 6 files changed, 174 insertions(+), 9 deletions(-) create mode 100644 tests/test_execution/test_out_of_core_fetch.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d0b3a14c..67fb564d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,13 @@ adheres to [Semantic Versioning](https://semver.org/). numpy 2.5+. Affects `impute="median"`/`"auto"`, the default missing-value engine, `fill_missing`, MissForest seeding and seasonal time-series imputation. +- `engine="duckdb"` with `output_format="arrow"` or `"polars"` now fetches the + result directly in that format instead of materializing a pandas frame with + `fetchdf()` and converting it again (#52). Arrow output keeps DuckDB's + column types (for example `DECIMAL` stays `decimal128`). +- The Polars engine's duplicate-detection and dedup row counts now run through + the streaming collect path instead of re-evaluating the whole plan in memory + (#53). ## [2.0.0] - 2026-07-20 diff --git a/src/freshdata/execution/__init__.py b/src/freshdata/execution/__init__.py index a36a29a2..dd0de9b4 100644 --- a/src/freshdata/execution/__init__.py +++ b/src/freshdata/execution/__init__.py @@ -125,9 +125,11 @@ def _convert_output(frame: Any, output_format: str) -> Any: from ._lazy import require_pyarrow require_pyarrow() - if is_pandas: - import pyarrow as pa + import pyarrow as pa + if isinstance(frame, pa.Table): + return frame # fetched as Arrow by the backend (DuckDB) + if is_pandas: return pa.Table.from_pandas(frame, preserve_index=False) return frame.to_arrow() # polars diff --git a/src/freshdata/execution/_report.py b/src/freshdata/execution/_report.py index 97cb1c3e..f6eca945 100644 --- a/src/freshdata/execution/_report.py +++ b/src/freshdata/execution/_report.py @@ -54,6 +54,16 @@ def _frame_stats(frame: Any) -> tuple[int, int, int, int]: except ImportError: # pragma: no cover pass + # arrow table (DuckDB fetches Arrow directly for output_format="arrow") + try: + import pyarrow as pa + + if isinstance(frame, pa.Table): + missing = sum(column.null_count for column in frame.columns) + return frame.num_rows, frame.num_columns, int(missing), int(frame.nbytes) + except ImportError: # pragma: no cover + pass + raise TypeError(f"cannot compute stats for {type(frame).__name__}") diff --git a/src/freshdata/execution/backends/_duckdb.py b/src/freshdata/execution/backends/_duckdb.py index e4241354..3a7d7cbc 100644 --- a/src/freshdata/execution/backends/_duckdb.py +++ b/src/freshdata/execution/backends/_duckdb.py @@ -124,14 +124,25 @@ def execute( meta = MetadataScanner.from_duckdb(conn, _TABLE) report = init_report(meta, self._memory_before(source)) report.backend = "duckdb" - cleaned = self._run_sql_pipeline( - conn, meta, plan, config, report, materialize=not native + relation = self._run_sql_pipeline( + conn, meta, plan, config, report, materialize=False ) if native: + cleaned = relation # The relation is tied to this connection; keep it open so the # caller can stream from it. Closing here would invalidate it. _keep_native_relation_connection_alive(cleaned, conn) close_conn = False + elif engine_config.output_format == "arrow" and not config.semantic_enabled: + # Fetch straight into the requested format instead of building a + # pandas frame and converting it again (the native semantic stage + # only accepts pandas/polars/relations, so it keeps fetchdf). + to_arrow = getattr(relation, "to_arrow_table", None) + cleaned = to_arrow() if to_arrow else relation.fetch_arrow_table() + elif engine_config.output_format == "polars" and not config.semantic_enabled: + cleaned = relation.pl() + else: + cleaned = relation.fetchdf() finally: if close_conn: conn.close() diff --git a/src/freshdata/execution/backends/_polars.py b/src/freshdata/execution/backends/_polars.py index 098b9958..e14043eb 100644 --- a/src/freshdata/execution/backends/_polars.py +++ b/src/freshdata/execution/backends/_polars.py @@ -436,7 +436,9 @@ def _stage_drop_duplicates( pl: Any, engine_config: EngineConfig | None = None, ) -> Any: - n_before = int(lf.select(pl.len()).collect().item()) + # Counts go through the streaming engine: a plain .collect() re-evaluates + # the whole upstream plan in memory, defeating out-of-core execution. + n_before = self._count(lf, engine_config, pl) if n_before < 1: return lf subset = list(config.duplicate_subset) if config.duplicate_subset is not None else None @@ -452,7 +454,7 @@ def _stage_drop_duplicates( if not config.drop_duplicates: # Detection-only default: count duplicates, report, keep every row. n_unique = int( - lf.unique(subset=subset).select(pl.len()).collect().item() + self._count(lf.unique(subset=subset), engine_config, pl) ) report_detected_duplicates( n_before - n_unique, n_before, config, report, subset=subset @@ -482,7 +484,7 @@ def _stage_drop_duplicates( deduped = lf.unique( subset=subset, keep=config.duplicate_keep, maintain_order=maintain_order ) - n_after = int(deduped.select(pl.len()).collect().item()) + n_after = self._count(deduped, engine_config, pl) n_dup = n_before - n_after if n_dup <= 0: return lf @@ -507,8 +509,13 @@ def _stage_drop_duplicates( # -- collection --------------------------------------------------------- - def _collect(self, lf: Any, engine_config: EngineConfig, pl: Any) -> Any: - if not engine_config.streaming: + def _count(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> int: + """Row count of *lf*, evaluated through the (streaming) collect path.""" + return int(self._collect(lf.select(pl.len()), engine_config, pl).item()) + + def _collect(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> Any: + # No explicit engine config means the defaults, where streaming is on. + if engine_config is not None and not engine_config.streaming: return lf.collect() # Polars renamed the streaming switch across versions; try the modern # keyword first, then the legacy one, then a plain collect. diff --git a/tests/test_execution/test_out_of_core_fetch.py b/tests/test_execution/test_out_of_core_fetch.py new file mode 100644 index 00000000..12d9fc6a --- /dev/null +++ b/tests/test_execution/test_out_of_core_fetch.py @@ -0,0 +1,128 @@ +"""Native engines keep results out of pandas unless pandas is requested. + +* DuckDB fetches ``output_format="arrow"`` / ``"polars"`` directly instead of + building a pandas frame with ``fetchdf()`` and converting it again (#52). +* Polars dedup row counts go through the streaming collect path instead of a + plain in-memory ``collect()`` of the whole upstream plan (#53). +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +import freshdata as fd +from freshdata import EngineConfig + +duckdb = pytest.importorskip("duckdb") +pa = pytest.importorskip("pyarrow") +pl = pytest.importorskip("polars") + +from freshdata.execution.backends._polars import PolarsEngine # noqa: E402 + + +def _dupes_and_empty_rows() -> pd.DataFrame: + return pd.DataFrame( + { + "k": ["a", "a", None, "b", "b", "c", "a"], + "v": [1.0, 1.0, None, 2.0, 3.0, 4.0, 5.0], + } + ) + + +@pytest.fixture +def no_fetchdf(monkeypatch): + def fail(self, *args, **kwargs): + raise AssertionError("fetchdf() must not run for non-pandas output") + + monkeypatch.setattr(duckdb.DuckDBPyRelation, "fetchdf", fail) + monkeypatch.setattr(duckdb.DuckDBPyRelation, "df", fail) + + +@pytest.mark.parametrize( + ("output_format", "kind"), [("arrow", pa.Table), ("polars", pl.DataFrame)] +) +def test_duckdb_fetches_requested_format_directly( + no_fetchdf, small_df, native_config, output_format, kind +): + out, report = fd.clean( + small_df.copy(), config=native_config, engine="duckdb", + output_format=output_format, return_report=True, + ) + assert isinstance(out, kind) + assert report.backend == "duckdb" + assert report.rows_after == out.shape[0] + assert report.cols_after == out.shape[1] + + +@pytest.mark.parametrize("output_format", ["arrow", "polars"]) +def test_duckdb_direct_fetch_matches_pandas_output(small_df, native_config, output_format): + expected, expected_report = fd.clean( + small_df.copy(), config=native_config, engine="duckdb", return_report=True + ) + out, report = fd.clean( + small_df.copy(), config=native_config, engine="duckdb", + output_format=output_format, return_report=True, + ) + pd.testing.assert_frame_equal( + out.to_pandas().reset_index(drop=True), + expected.reset_index(drop=True), + check_dtype=False, + ) + assert report.missing_after == expected_report.missing_after + assert report.duplicates_removed == expected_report.duplicates_removed + + +@pytest.mark.parametrize("drop_duplicates", [True, False]) +@pytest.mark.parametrize("streaming", [True, False]) +def test_polars_dedup_counts_follow_the_engine_streaming_setting( + monkeypatch, native_config, drop_duplicates, streaming +): + engines: list[object] = [] + in_dedup = False + original_collect = pl.LazyFrame.collect + original_stage = PolarsEngine._stage_drop_duplicates + + def recording_collect(self, *args, **kwargs): + if in_dedup: + engines.append(kwargs.get("engine")) + return original_collect(self, *args, **kwargs) + + def tracked_stage(self, *args, **kwargs): + nonlocal in_dedup + in_dedup = True + try: + return original_stage(self, *args, **kwargs) + finally: + in_dedup = False + + monkeypatch.setattr(pl.LazyFrame, "collect", recording_collect) + monkeypatch.setattr(PolarsEngine, "_stage_drop_duplicates", tracked_stage) + fd.clean( + _dupes_and_empty_rows(), config=native_config, engine="polars", + engine_config=EngineConfig(engine="polars", streaming=streaming), + drop_duplicates=drop_duplicates, return_report=True, + ) + assert engines, "the dedup stage should count rows" + assert set(engines) == ({"streaming"} if streaming else {None}) + + +@pytest.mark.parametrize("keep", ["first", "last"]) +@pytest.mark.parametrize("subset", [None, ["k"]]) +def test_polars_dedup_after_empty_row_drop_matches_pandas(native_config, keep, subset): + options = {"drop_duplicates": True, "duplicate_keep": keep, "duplicate_subset": subset} + expected, expected_report = fd.clean( + _dupes_and_empty_rows(), config=native_config, engine="pandas", + return_report=True, **options, + ) + out, report = fd.clean( + _dupes_and_empty_rows(), config=native_config, engine="polars", + engine_config=EngineConfig(engine="polars", streaming_dedup=False), + return_report=True, **options, + ) + assert report.backend == "polars" + pd.testing.assert_frame_equal( + out.reset_index(drop=True), expected.reset_index(drop=True), check_dtype=False + ) + assert report.duplicates_removed == expected_report.duplicates_removed + assert report.rows_after == expected_report.rows_after