Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,18 @@ adheres to [Semantic Versioning](https://semver.org/).
previews before entering `model_context` in **all** privacy modes
(including `schema_only`); the local `report.problems` keeps the rich
previews.
- Out-of-core docs now match measured behavior: keeping a native handle
requires `fix_dtypes=False` **in addition to** `strategy="conservative"`
(dtype fixing runs sampled pandas heuristics and forces the recorded
fallback), and `output_format="polars-lazy"` defers only the *final*
materialization — pipeline stages currently collect intermediates
eagerly, so peak memory during cleaning matches eager output. The DuckDB
handle path is the measured lower-peak-memory route (#52, #53).

### Added
- `benchmarks/bench_outofcore.py`: subprocess-isolated peak-RSS evidence for
the four engine/output-format combinations on a generated parquet fixture
(per-scenario `ru_maxrss`, wall time, and the `materialized` flag).
- **AI Copilot (experimental)** — `freshdata.experimental.ai_copilot.analyze_dataset`:
deterministic, fully offline dataset analysis that returns a ranked problem
list (PII, policy violations, duplicates, missing values, mixed date
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ every new dataset and want an audit trail they can hand to a reviewer.
removes outliers blindly.
- **pandas-first, Polars-optional** — pandas + NumPy core; pass a Polars frame
and get one back, with optional Polars/DuckDB/Spark execution backends for
larger-than-memory data.
scaling beyond pandas (see `docs/backends.md` for the measured
out-of-core boundary per backend and output format).
- **CLI included** — `clean`, `plan`, `apply-plan`, `profile`, `learn`, and
`trust` subcommands for scripting and CI pipelines without writing Python.
- **Typed, tested, fast** — fully type-hinted (`py.typed`), vectorized, with a
Expand Down
147 changes: 147 additions & 0 deletions benchmarks/bench_outofcore.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
"""Peak-RSS evidence for the out-of-core claims (issues #52 / #53).

Measures, in an isolated subprocess per scenario, the peak resident set size
of cleaning a parquet fixture through each engine/output_format combination:

engine=duckdb output_format=pandas (default: materializes at the end)
engine=duckdb output_format=duckdb (un-fetched relation handle)
engine=polars output_format=pandas (default: materializes at the end)
engine=polars output_format=polars-lazy (un-collected LazyFrame handle)

The fixture is written batch-by-batch so generating it never holds the whole
frame in memory. Each scenario runs in its own subprocess so allocator reuse
cannot blur the numbers; peak RSS is read from ``resource.ru_maxrss`` at exit.

Usage::

python benchmarks/bench_outofcore.py --rows 2000000

Not part of the test suite: RSS numbers are machine-specific evidence, not
assertions.
"""

from __future__ import annotations

import argparse
import json
import subprocess
import sys
import tempfile
import time
from pathlib import Path

SCENARIOS = [
("duckdb", "pandas"),
("duckdb", "duckdb"),
("polars", "pandas"),
("polars", "polars-lazy"),
]

_BATCH = 250_000


def make_fixture(path: Path, rows: int) -> None:
"""Write a mixed-dtype parquet fixture in bounded batches."""
import numpy as np
import pandas as pd
import pyarrow as pa
import pyarrow.parquet as pq

writer = None
rng = np.random.default_rng(0)
written = 0
while written < rows:
n = min(_BATCH, rows - written)
idx = np.arange(written, written + n)
df = pd.DataFrame(
{
"id": idx,
"amount": rng.normal(100.0, 25.0, n),
"count": rng.integers(0, 1_000, n),
"city": pd.Series(idx % 500).map("city_{:03d}".format),
"note": pd.Series(idx % 1_000).map("note text {:04d} padding".format),
}
)
table = pa.Table.from_pandas(df, preserve_index=False)
if writer is None:
writer = pq.ParquetWriter(path, table.schema)
writer.write_table(table)
written += n
if writer is not None:
writer.close()


def child(engine: str, output_format: str, fixture: str) -> None:
"""Run one clean in this process and print a JSON result line."""
import resource

import freshdata as fd

start = time.perf_counter()
# fix_dtypes uses sampled pandas heuristics, so it forces a fallback that
# materializes the frame; disabling it is part of the native-handle recipe.
result, report = fd.clean(
fixture,
engine=engine,
output_format=output_format,
strategy="conservative",
fix_dtypes=False,
return_report=True,
)
elapsed = time.perf_counter() - start
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
if sys.platform != "darwin": # linux reports KiB, macOS bytes
peak *= 1024
print(
json.dumps(
{
"engine": engine,
"output_format": output_format,
"materialized": report.materialized,
"peak_rss_mb": round(peak / 1e6, 1),
"seconds": round(elapsed, 2),
"result_type": type(result).__name__,
}
)
)


def main() -> None:
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument("--rows", type=int, default=2_000_000)
ap.add_argument("--fixture", help="reuse an existing parquet fixture")
ap.add_argument("--_child", nargs=3, help=argparse.SUPPRESS)
args = ap.parse_args()

if args._child:
child(*args._child)
return

with tempfile.TemporaryDirectory() as tmp:
fixture = args.fixture or str(Path(tmp) / "fixture.parquet")
if not args.fixture:
print(f"generating {args.rows:,}-row fixture ...", file=sys.stderr)
make_fixture(Path(fixture), args.rows)

results = []
for engine, fmt in SCENARIOS:
proc = subprocess.run(
[sys.executable, __file__, "--_child", engine, fmt, fixture],
capture_output=True,
text=True,
check=False,
)
if proc.returncode != 0:
print(f"{engine}/{fmt} FAILED:\n{proc.stderr}", file=sys.stderr)
continue
results.append(json.loads(proc.stdout.strip().splitlines()[-1]))

print(f"\n{'engine':<8} {'output_format':<13} {'materialized':<13} "
f"{'peak RSS (MB)':>13} {'seconds':>8}")
for r in results:
print(f"{r['engine']:<8} {r['output_format']:<13} "
f"{str(r['materialized']):<13} {r['peak_rss_mb']:>13} {r['seconds']:>8}")


if __name__ == "__main__":
main()
16 changes: 12 additions & 4 deletions docs/backends.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,14 +36,22 @@ memory:
| `"duckdb"` | `DuckDBPyRelation` (un-fetched) | **No** — you call `.fetchdf()`/`.arrow()` |
| `"polars-lazy"` | `pl.LazyFrame` (un-collected) | **No** — you call `.collect()` |

Only the `"duckdb"` and `"polars-lazy"` handles are larger-than-RAM safe end to
end: nothing is fetched/collected until *you* ask. When a clean returns a native
Neither handle fetches/collects the *result* until you ask, but they are not
equal during the pipeline: the DuckDB path keeps peak memory well below the
eager equivalent, while the Polars pipeline currently collects intermediates
eagerly between stages, so `"polars-lazy"` defers only the final
materialization — peak RSS during cleaning is comparable to eager output
(measured; reproduce with `python benchmarks/bench_outofcore.py`). When a clean returns a native
handle, `report.materialized` is `False` and `report.summary()` says so plainly.
If the requested strategy needs the pandas decision engine (e.g. `balanced`/
`aggressive` imputation, dtype heuristics), the backend transparently falls back
to pandas — that fallback is recorded in `report.fallback_events`, and the result
is materialized. Use `strategy="conservative"` (deterministic representation
repair + structural reduction) to keep the native handle.
is materialized. To keep the native handle use `strategy="conservative"`
(deterministic representation repair + structural reduction) **and**
`fix_dtypes=False` — dtype fixing relies on sampled pandas heuristics and
forces the fallback even under `conservative`. Peak-RSS evidence for the
four engine/output combinations is reproducible with
`python benchmarks/bench_outofcore.py`.

**Semantic cleaning stays native too.** When `semantic_mode` is enabled on a
Polars/DuckDB engine, the semantic stage runs over a *natively extracted*
Expand Down
9 changes: 6 additions & 3 deletions docs/limitations.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,18 @@ The short version:
| `fd.clean(df)` (default pandas) | In-memory; the whole frame is in RAM. |
| `engine="polars"` / `"duckdb"` / `"spark"`, `output_format="pandas"` (default) | Scales out **during** the pipeline (spill-to-disk), then **materializes** the result into a pandas frame. |
| `output_format="duckdb"` | Returns an un-fetched `DuckDBPyRelation` — **not materialized**; you call `.fetchdf()`. |
| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — **not materialized**; you call `.collect()`. |
| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — the **result** is not materialized (you call `.collect()`), but pipeline stages currently collect intermediates eagerly, so peak memory *during* cleaning is comparable to eager output. DuckDB is the lower-peak-memory native path today (measured: `python benchmarks/bench_outofcore.py`). |
| `StreamingCleaner` / `fd.clean_timeseries(..., stream=...)` | Genuinely out-of-core: bounded micro-batches, running statistics, never concatenated. |

`report.materialized` is `False` whenever a native handle is returned, and
`report.summary()` says so. If a strategy needs the pandas decision engine
(`balanced` / `aggressive` imputation, dtype heuristics), the native backends
transparently fall back to pandas — recorded in `report.fallback_events` — and
the result is materialized. Use `strategy="conservative"` to keep the native
handle. Streaming holds memory flat by design: bounded reservoirs and counters,
the result is materialized. To keep the native handle use
`strategy="conservative"` **and** `fix_dtypes=False` — dtype fixing uses
sampled pandas heuristics and forces the fallback even under `conservative`
(measure it yourself: `python benchmarks/bench_outofcore.py`).
Streaming holds memory flat by design: bounded reservoirs and counters,
a recent-window (not global) dedup, and no cross-batch concatenation.

---
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -346,6 +346,7 @@ ignore = [
# `import freshdata` stay dependency-free.
"src/freshdata/streaming/*" = ["PLC0415"]
"benchmarks/bench_streaming.py" = ["PLC0415"]
"benchmarks/bench_outofcore.py" = ["PLC0415"]
"benchmarks/bench_native_semantic.py" = ["PLC0415"]
"benchmarks/bench_report.py" = ["PLC0415"]
"benchmarks/bench_freshcore.py" = ["E402", "I001"]
Expand Down
Loading