Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,13 @@ adheres to [Semantic Versioning](https://semver.org/).
numpy 2.5+. Affects `impute="median"`/`"auto"`, the default missing-value
engine, `fill_missing`, MissForest seeding and seasonal time-series
imputation.
- `engine="duckdb"` with `output_format="arrow"` or `"polars"` now fetches the
result directly in that format instead of materializing a pandas frame with
`fetchdf()` and converting it again (#52). Arrow output keeps DuckDB's
column types (for example `DECIMAL` stays `decimal128`).
- The Polars engine's duplicate-detection and dedup row counts now run through
the streaming collect path instead of re-evaluating the whole plan in memory
(#53).

## [2.0.0] - 2026-07-20

Expand Down
6 changes: 4 additions & 2 deletions src/freshdata/execution/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,11 @@ def _convert_output(frame: Any, output_format: str) -> Any:
from ._lazy import require_pyarrow

require_pyarrow()
if is_pandas:
import pyarrow as pa
import pyarrow as pa

if isinstance(frame, pa.Table):
return frame # fetched as Arrow by the backend (DuckDB)
if is_pandas:
return pa.Table.from_pandas(frame, preserve_index=False)
return frame.to_arrow() # polars

Expand Down
10 changes: 10 additions & 0 deletions src/freshdata/execution/_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,16 @@ def _frame_stats(frame: Any) -> tuple[int, int, int, int]:
except ImportError: # pragma: no cover
pass

# arrow table (DuckDB fetches Arrow directly for output_format="arrow")
try:
import pyarrow as pa

if isinstance(frame, pa.Table):
missing = sum(column.null_count for column in frame.columns)
return frame.num_rows, frame.num_columns, int(missing), int(frame.nbytes)
except ImportError: # pragma: no cover
pass

raise TypeError(f"cannot compute stats for {type(frame).__name__}")


Expand Down
15 changes: 13 additions & 2 deletions src/freshdata/execution/backends/_duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,14 +124,25 @@ def execute(
meta = MetadataScanner.from_duckdb(conn, _TABLE)
report = init_report(meta, self._memory_before(source))
report.backend = "duckdb"
cleaned = self._run_sql_pipeline(
conn, meta, plan, config, report, materialize=not native
relation = self._run_sql_pipeline(
conn, meta, plan, config, report, materialize=False
)
if native:
cleaned = relation
# The relation is tied to this connection; keep it open so the
# caller can stream from it. Closing here would invalidate it.
_keep_native_relation_connection_alive(cleaned, conn)
close_conn = False
elif engine_config.output_format == "arrow" and not config.semantic_enabled:
# Fetch straight into the requested format instead of building a
# pandas frame and converting it again (the native semantic stage
# only accepts pandas/polars/relations, so it keeps fetchdf).
to_arrow = getattr(relation, "to_arrow_table", None)
cleaned = to_arrow() if to_arrow else relation.fetch_arrow_table()
elif engine_config.output_format == "polars" and not config.semantic_enabled:
cleaned = relation.pl()
else:
cleaned = relation.fetchdf()
finally:
if close_conn:
conn.close()
Expand Down
17 changes: 12 additions & 5 deletions src/freshdata/execution/backends/_polars.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,9 @@ def _stage_drop_duplicates(
pl: Any,
engine_config: EngineConfig | None = None,
) -> Any:
n_before = int(lf.select(pl.len()).collect().item())
# Counts go through the streaming engine: a plain .collect() re-evaluates
# the whole upstream plan in memory, defeating out-of-core execution.
n_before = self._count(lf, engine_config, pl)
if n_before < 1:
return lf
subset = list(config.duplicate_subset) if config.duplicate_subset is not None else None
Expand All @@ -452,7 +454,7 @@ def _stage_drop_duplicates(
if not config.drop_duplicates:
# Detection-only default: count duplicates, report, keep every row.
n_unique = int(
lf.unique(subset=subset).select(pl.len()).collect().item()
self._count(lf.unique(subset=subset), engine_config, pl)
)
report_detected_duplicates(
n_before - n_unique, n_before, config, report, subset=subset
Expand Down Expand Up @@ -482,7 +484,7 @@ def _stage_drop_duplicates(
deduped = lf.unique(
subset=subset, keep=config.duplicate_keep, maintain_order=maintain_order
)
n_after = int(deduped.select(pl.len()).collect().item())
n_after = self._count(deduped, engine_config, pl)
n_dup = n_before - n_after
if n_dup <= 0:
return lf
Expand All @@ -507,8 +509,13 @@ def _stage_drop_duplicates(

# -- collection ---------------------------------------------------------

def _collect(self, lf: Any, engine_config: EngineConfig, pl: Any) -> Any:
if not engine_config.streaming:
def _count(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> int:
"""Row count of *lf*, evaluated through the (streaming) collect path."""
return int(self._collect(lf.select(pl.len()), engine_config, pl).item())

def _collect(self, lf: Any, engine_config: EngineConfig | None, pl: Any) -> Any:
# No explicit engine config means the defaults, where streaming is on.
if engine_config is not None and not engine_config.streaming:
return lf.collect()
# Polars renamed the streaming switch across versions; try the modern
# keyword first, then the legacy one, then a plain collect.
Expand Down
128 changes: 128 additions & 0 deletions tests/test_execution/test_out_of_core_fetch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
"""Native engines keep results out of pandas unless pandas is requested.

* DuckDB fetches ``output_format="arrow"`` / ``"polars"`` directly instead of
building a pandas frame with ``fetchdf()`` and converting it again (#52).
* Polars dedup row counts go through the streaming collect path instead of a
plain in-memory ``collect()`` of the whole upstream plan (#53).
"""

from __future__ import annotations

import pandas as pd
import pytest

import freshdata as fd
from freshdata import EngineConfig

duckdb = pytest.importorskip("duckdb")
pa = pytest.importorskip("pyarrow")
pl = pytest.importorskip("polars")

from freshdata.execution.backends._polars import PolarsEngine # noqa: E402


def _dupes_and_empty_rows() -> pd.DataFrame:
return pd.DataFrame(
{
"k": ["a", "a", None, "b", "b", "c", "a"],
"v": [1.0, 1.0, None, 2.0, 3.0, 4.0, 5.0],
}
)


@pytest.fixture
def no_fetchdf(monkeypatch):
def fail(self, *args, **kwargs):
raise AssertionError("fetchdf() must not run for non-pandas output")

monkeypatch.setattr(duckdb.DuckDBPyRelation, "fetchdf", fail)
monkeypatch.setattr(duckdb.DuckDBPyRelation, "df", fail)


@pytest.mark.parametrize(
("output_format", "kind"), [("arrow", pa.Table), ("polars", pl.DataFrame)]
)
def test_duckdb_fetches_requested_format_directly(
no_fetchdf, small_df, native_config, output_format, kind
):
out, report = fd.clean(
small_df.copy(), config=native_config, engine="duckdb",
output_format=output_format, return_report=True,
)
assert isinstance(out, kind)
assert report.backend == "duckdb"
assert report.rows_after == out.shape[0]
assert report.cols_after == out.shape[1]


@pytest.mark.parametrize("output_format", ["arrow", "polars"])
def test_duckdb_direct_fetch_matches_pandas_output(small_df, native_config, output_format):
expected, expected_report = fd.clean(
small_df.copy(), config=native_config, engine="duckdb", return_report=True
)
out, report = fd.clean(
small_df.copy(), config=native_config, engine="duckdb",
output_format=output_format, return_report=True,
)
pd.testing.assert_frame_equal(
out.to_pandas().reset_index(drop=True),
expected.reset_index(drop=True),
check_dtype=False,
)
assert report.missing_after == expected_report.missing_after
assert report.duplicates_removed == expected_report.duplicates_removed


@pytest.mark.parametrize("drop_duplicates", [True, False])
@pytest.mark.parametrize("streaming", [True, False])
def test_polars_dedup_counts_follow_the_engine_streaming_setting(
monkeypatch, native_config, drop_duplicates, streaming
):
engines: list[object] = []
in_dedup = False
original_collect = pl.LazyFrame.collect
original_stage = PolarsEngine._stage_drop_duplicates

def recording_collect(self, *args, **kwargs):
if in_dedup:
engines.append(kwargs.get("engine"))
return original_collect(self, *args, **kwargs)

def tracked_stage(self, *args, **kwargs):
nonlocal in_dedup
in_dedup = True
try:
return original_stage(self, *args, **kwargs)
finally:
in_dedup = False

monkeypatch.setattr(pl.LazyFrame, "collect", recording_collect)
monkeypatch.setattr(PolarsEngine, "_stage_drop_duplicates", tracked_stage)
fd.clean(
_dupes_and_empty_rows(), config=native_config, engine="polars",
engine_config=EngineConfig(engine="polars", streaming=streaming),
drop_duplicates=drop_duplicates, return_report=True,
)
assert engines, "the dedup stage should count rows"
assert set(engines) == ({"streaming"} if streaming else {None})


@pytest.mark.parametrize("keep", ["first", "last"])
@pytest.mark.parametrize("subset", [None, ["k"]])
def test_polars_dedup_after_empty_row_drop_matches_pandas(native_config, keep, subset):
options = {"drop_duplicates": True, "duplicate_keep": keep, "duplicate_subset": subset}
expected, expected_report = fd.clean(
_dupes_and_empty_rows(), config=native_config, engine="pandas",
return_report=True, **options,
)
out, report = fd.clean(
_dupes_and_empty_rows(), config=native_config, engine="polars",
engine_config=EngineConfig(engine="polars", streaming_dedup=False),
return_report=True, **options,
)
assert report.backend == "polars"
pd.testing.assert_frame_equal(
out.reset_index(drop=True), expected.reset_index(drop=True), check_dtype=False
)
assert report.duplicates_removed == expected_report.duplicates_removed
assert report.rows_after == expected_report.rows_after
Loading