From 860c0152489ce27c6fd43819f29b71cd85fb3a54 Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Sat, 11 Jul 2026 10:12:34 +0530 Subject: [PATCH] docs+bench: measured out-of-core evidence; correct native-handle recipe and polars-lazy claims (#52, #53) New benchmarks/bench_outofcore.py measures peak RSS (subprocess-isolated ru_maxrss) for the four engine/output-format combinations on a generated parquet fixture. Measured at 5M rows: duckdb/pandas 1771 MB materialized duckdb/duckdb 723 MB native handle (2.4x lower peak) polars/pandas 2195 MB materialized polars/polars-lazy 2222 MB handle, but NO peak reduction Two doc corrections the measurements forced: - The documented native-handle recipe was incomplete: default fix_dtypes=True runs sampled pandas heuristics and triggers the (recorded) fallback even under strategy='conservative', materializing the result. backends.md/limitations.md now state the working recipe: strategy='conservative' AND fix_dtypes=False. - polars-lazy was described as larger-than-RAM safe end to end; in fact the Polars pipeline collects intermediates eagerly between stages, so the lazy handle defers only the final materialization and peak RSS during cleaning matches eager output. Docs now state the per-backend boundary; README points at the measured boundary instead of a flat larger-than-memory claim. No code behavior changes. CLAIM_REGISTRY claims untouched. --- CHANGELOG.md | 10 +++ README.md | 3 +- benchmarks/bench_outofcore.py | 147 ++++++++++++++++++++++++++++++++++ docs/backends.md | 16 +++- docs/limitations.md | 9 ++- pyproject.toml | 1 + 6 files changed, 178 insertions(+), 8 deletions(-) create mode 100644 benchmarks/bench_outofcore.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c9596dda..ccb1db76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,18 @@ adheres to [Semantic Versioning](https://semver.org/). previews before entering `model_context` in **all** privacy modes (including `schema_only`); the local `report.problems` keeps the rich previews. +- Out-of-core docs now match measured behavior: keeping a native handle + requires `fix_dtypes=False` **in addition to** `strategy="conservative"` + (dtype fixing runs sampled pandas heuristics and forces the recorded + fallback), and `output_format="polars-lazy"` defers only the *final* + materialization — pipeline stages currently collect intermediates + eagerly, so peak memory during cleaning matches eager output. The DuckDB + handle path is the measured lower-peak-memory route (#52, #53). ### Added +- `benchmarks/bench_outofcore.py`: subprocess-isolated peak-RSS evidence for + the four engine/output-format combinations on a generated parquet fixture + (per-scenario `ru_maxrss`, wall time, and the `materialized` flag). - **AI Copilot (experimental)** — `freshdata.experimental.ai_copilot.analyze_dataset`: deterministic, fully offline dataset analysis that returns a ranked problem list (PII, policy violations, duplicates, missing values, mixed date diff --git a/README.md b/README.md index d4056217..ad71c53c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ every new dataset and want an audit trail they can hand to a reviewer. removes outliers blindly. - **pandas-first, Polars-optional** — pandas + NumPy core; pass a Polars frame and get one back, with optional Polars/DuckDB/Spark execution backends for - larger-than-memory data. + scaling beyond pandas (see `docs/backends.md` for the measured + out-of-core boundary per backend and output format). - **CLI included** — `clean`, `plan`, `apply-plan`, `profile`, `learn`, and `trust` subcommands for scripting and CI pipelines without writing Python. - **Typed, tested, fast** — fully type-hinted (`py.typed`), vectorized, with a diff --git a/benchmarks/bench_outofcore.py b/benchmarks/bench_outofcore.py new file mode 100644 index 00000000..192feca1 --- /dev/null +++ b/benchmarks/bench_outofcore.py @@ -0,0 +1,147 @@ +"""Peak-RSS evidence for the out-of-core claims (issues #52 / #53). + +Measures, in an isolated subprocess per scenario, the peak resident set size +of cleaning a parquet fixture through each engine/output_format combination: + + engine=duckdb output_format=pandas (default: materializes at the end) + engine=duckdb output_format=duckdb (un-fetched relation handle) + engine=polars output_format=pandas (default: materializes at the end) + engine=polars output_format=polars-lazy (un-collected LazyFrame handle) + +The fixture is written batch-by-batch so generating it never holds the whole +frame in memory. Each scenario runs in its own subprocess so allocator reuse +cannot blur the numbers; peak RSS is read from ``resource.ru_maxrss`` at exit. + +Usage:: + + python benchmarks/bench_outofcore.py --rows 2000000 + +Not part of the test suite: RSS numbers are machine-specific evidence, not +assertions. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +SCENARIOS = [ + ("duckdb", "pandas"), + ("duckdb", "duckdb"), + ("polars", "pandas"), + ("polars", "polars-lazy"), +] + +_BATCH = 250_000 + + +def make_fixture(path: Path, rows: int) -> None: + """Write a mixed-dtype parquet fixture in bounded batches.""" + import numpy as np + import pandas as pd + import pyarrow as pa + import pyarrow.parquet as pq + + writer = None + rng = np.random.default_rng(0) + written = 0 + while written < rows: + n = min(_BATCH, rows - written) + idx = np.arange(written, written + n) + df = pd.DataFrame( + { + "id": idx, + "amount": rng.normal(100.0, 25.0, n), + "count": rng.integers(0, 1_000, n), + "city": pd.Series(idx % 500).map("city_{:03d}".format), + "note": pd.Series(idx % 1_000).map("note text {:04d} padding".format), + } + ) + table = pa.Table.from_pandas(df, preserve_index=False) + if writer is None: + writer = pq.ParquetWriter(path, table.schema) + writer.write_table(table) + written += n + if writer is not None: + writer.close() + + +def child(engine: str, output_format: str, fixture: str) -> None: + """Run one clean in this process and print a JSON result line.""" + import resource + + import freshdata as fd + + start = time.perf_counter() + # fix_dtypes uses sampled pandas heuristics, so it forces a fallback that + # materializes the frame; disabling it is part of the native-handle recipe. + result, report = fd.clean( + fixture, + engine=engine, + output_format=output_format, + strategy="conservative", + fix_dtypes=False, + return_report=True, + ) + elapsed = time.perf_counter() - start + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if sys.platform != "darwin": # linux reports KiB, macOS bytes + peak *= 1024 + print( + json.dumps( + { + "engine": engine, + "output_format": output_format, + "materialized": report.materialized, + "peak_rss_mb": round(peak / 1e6, 1), + "seconds": round(elapsed, 2), + "result_type": type(result).__name__, + } + ) + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--rows", type=int, default=2_000_000) + ap.add_argument("--fixture", help="reuse an existing parquet fixture") + ap.add_argument("--_child", nargs=3, help=argparse.SUPPRESS) + args = ap.parse_args() + + if args._child: + child(*args._child) + return + + with tempfile.TemporaryDirectory() as tmp: + fixture = args.fixture or str(Path(tmp) / "fixture.parquet") + if not args.fixture: + print(f"generating {args.rows:,}-row fixture ...", file=sys.stderr) + make_fixture(Path(fixture), args.rows) + + results = [] + for engine, fmt in SCENARIOS: + proc = subprocess.run( + [sys.executable, __file__, "--_child", engine, fmt, fixture], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + print(f"{engine}/{fmt} FAILED:\n{proc.stderr}", file=sys.stderr) + continue + results.append(json.loads(proc.stdout.strip().splitlines()[-1])) + + print(f"\n{'engine':<8} {'output_format':<13} {'materialized':<13} " + f"{'peak RSS (MB)':>13} {'seconds':>8}") + for r in results: + print(f"{r['engine']:<8} {r['output_format']:<13} " + f"{str(r['materialized']):<13} {r['peak_rss_mb']:>13} {r['seconds']:>8}") + + +if __name__ == "__main__": + main() diff --git a/docs/backends.md b/docs/backends.md index a936de9d..110022f8 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -36,14 +36,22 @@ memory: | `"duckdb"` | `DuckDBPyRelation` (un-fetched) | **No** — you call `.fetchdf()`/`.arrow()` | | `"polars-lazy"` | `pl.LazyFrame` (un-collected) | **No** — you call `.collect()` | -Only the `"duckdb"` and `"polars-lazy"` handles are larger-than-RAM safe end to -end: nothing is fetched/collected until *you* ask. When a clean returns a native +Neither handle fetches/collects the *result* until you ask, but they are not +equal during the pipeline: the DuckDB path keeps peak memory well below the +eager equivalent, while the Polars pipeline currently collects intermediates +eagerly between stages, so `"polars-lazy"` defers only the final +materialization — peak RSS during cleaning is comparable to eager output +(measured; reproduce with `python benchmarks/bench_outofcore.py`). When a clean returns a native handle, `report.materialized` is `False` and `report.summary()` says so plainly. If the requested strategy needs the pandas decision engine (e.g. `balanced`/ `aggressive` imputation, dtype heuristics), the backend transparently falls back to pandas — that fallback is recorded in `report.fallback_events`, and the result -is materialized. Use `strategy="conservative"` (deterministic representation -repair + structural reduction) to keep the native handle. +is materialized. To keep the native handle use `strategy="conservative"` +(deterministic representation repair + structural reduction) **and** +`fix_dtypes=False` — dtype fixing relies on sampled pandas heuristics and +forces the fallback even under `conservative`. Peak-RSS evidence for the +four engine/output combinations is reproducible with +`python benchmarks/bench_outofcore.py`. **Semantic cleaning stays native too.** When `semantic_mode` is enabled on a Polars/DuckDB engine, the semantic stage runs over a *natively extracted* diff --git a/docs/limitations.md b/docs/limitations.md index 396b4ee6..c729d4a5 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -34,15 +34,18 @@ The short version: | `fd.clean(df)` (default pandas) | In-memory; the whole frame is in RAM. | | `engine="polars"` / `"duckdb"` / `"spark"`, `output_format="pandas"` (default) | Scales out **during** the pipeline (spill-to-disk), then **materializes** the result into a pandas frame. | | `output_format="duckdb"` | Returns an un-fetched `DuckDBPyRelation` — **not materialized**; you call `.fetchdf()`. | -| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — **not materialized**; you call `.collect()`. | +| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — the **result** is not materialized (you call `.collect()`), but pipeline stages currently collect intermediates eagerly, so peak memory *during* cleaning is comparable to eager output. DuckDB is the lower-peak-memory native path today (measured: `python benchmarks/bench_outofcore.py`). | | `StreamingCleaner` / `fd.clean_timeseries(..., stream=...)` | Genuinely out-of-core: bounded micro-batches, running statistics, never concatenated. | `report.materialized` is `False` whenever a native handle is returned, and `report.summary()` says so. If a strategy needs the pandas decision engine (`balanced` / `aggressive` imputation, dtype heuristics), the native backends transparently fall back to pandas — recorded in `report.fallback_events` — and -the result is materialized. Use `strategy="conservative"` to keep the native -handle. Streaming holds memory flat by design: bounded reservoirs and counters, +the result is materialized. To keep the native handle use +`strategy="conservative"` **and** `fix_dtypes=False` — dtype fixing uses +sampled pandas heuristics and forces the fallback even under `conservative` +(measure it yourself: `python benchmarks/bench_outofcore.py`). +Streaming holds memory flat by design: bounded reservoirs and counters, a recent-window (not global) dedup, and no cross-batch concatenation. --- diff --git a/pyproject.toml b/pyproject.toml index a685ddfd..5f2119d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -346,6 +346,7 @@ ignore = [ # `import freshdata` stay dependency-free. "src/freshdata/streaming/*" = ["PLC0415"] "benchmarks/bench_streaming.py" = ["PLC0415"] +"benchmarks/bench_outofcore.py" = ["PLC0415"] "benchmarks/bench_native_semantic.py" = ["PLC0415"] "benchmarks/bench_report.py" = ["PLC0415"] "benchmarks/bench_freshcore.py" = ["E402", "I001"]