diff --git a/CHANGELOG.md b/CHANGELOG.md index c9596dda..ccb1db76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,8 +32,18 @@ adheres to [Semantic Versioning](https://semver.org/). previews before entering `model_context` in **all** privacy modes (including `schema_only`); the local `report.problems` keeps the rich previews. +- Out-of-core docs now match measured behavior: keeping a native handle + requires `fix_dtypes=False` **in addition to** `strategy="conservative"` + (dtype fixing runs sampled pandas heuristics and forces the recorded + fallback), and `output_format="polars-lazy"` defers only the *final* + materialization — pipeline stages currently collect intermediates + eagerly, so peak memory during cleaning matches eager output. The DuckDB + handle path is the measured lower-peak-memory route (#52, #53). ### Added +- `benchmarks/bench_outofcore.py`: subprocess-isolated peak-RSS evidence for + the four engine/output-format combinations on a generated parquet fixture + (per-scenario `ru_maxrss`, wall time, and the `materialized` flag). - **AI Copilot (experimental)** — `freshdata.experimental.ai_copilot.analyze_dataset`: deterministic, fully offline dataset analysis that returns a ranked problem list (PII, policy violations, duplicates, missing values, mixed date diff --git a/README.md b/README.md index d4056217..ad71c53c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ every new dataset and want an audit trail they can hand to a reviewer. removes outliers blindly. - **pandas-first, Polars-optional** — pandas + NumPy core; pass a Polars frame and get one back, with optional Polars/DuckDB/Spark execution backends for - larger-than-memory data. + scaling beyond pandas (see `docs/backends.md` for the measured + out-of-core boundary per backend and output format). - **CLI included** — `clean`, `plan`, `apply-plan`, `profile`, `learn`, and `trust` subcommands for scripting and CI pipelines without writing Python. - **Typed, tested, fast** — fully type-hinted (`py.typed`), vectorized, with a diff --git a/benchmarks/bench_outofcore.py b/benchmarks/bench_outofcore.py new file mode 100644 index 00000000..192feca1 --- /dev/null +++ b/benchmarks/bench_outofcore.py @@ -0,0 +1,147 @@ +"""Peak-RSS evidence for the out-of-core claims (issues #52 / #53). + +Measures, in an isolated subprocess per scenario, the peak resident set size +of cleaning a parquet fixture through each engine/output_format combination: + + engine=duckdb output_format=pandas (default: materializes at the end) + engine=duckdb output_format=duckdb (un-fetched relation handle) + engine=polars output_format=pandas (default: materializes at the end) + engine=polars output_format=polars-lazy (un-collected LazyFrame handle) + +The fixture is written batch-by-batch so generating it never holds the whole +frame in memory. Each scenario runs in its own subprocess so allocator reuse +cannot blur the numbers; peak RSS is read from ``resource.ru_maxrss`` at exit. + +Usage:: + + python benchmarks/bench_outofcore.py --rows 2000000 + +Not part of the test suite: RSS numbers are machine-specific evidence, not +assertions. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +SCENARIOS = [ + ("duckdb", "pandas"), + ("duckdb", "duckdb"), + ("polars", "pandas"), + ("polars", "polars-lazy"), +] + +_BATCH = 250_000 + + +def make_fixture(path: Path, rows: int) -> None: + """Write a mixed-dtype parquet fixture in bounded batches.""" + import numpy as np + import pandas as pd + import pyarrow as pa + import pyarrow.parquet as pq + + writer = None + rng = np.random.default_rng(0) + written = 0 + while written < rows: + n = min(_BATCH, rows - written) + idx = np.arange(written, written + n) + df = pd.DataFrame( + { + "id": idx, + "amount": rng.normal(100.0, 25.0, n), + "count": rng.integers(0, 1_000, n), + "city": pd.Series(idx % 500).map("city_{:03d}".format), + "note": pd.Series(idx % 1_000).map("note text {:04d} padding".format), + } + ) + table = pa.Table.from_pandas(df, preserve_index=False) + if writer is None: + writer = pq.ParquetWriter(path, table.schema) + writer.write_table(table) + written += n + if writer is not None: + writer.close() + + +def child(engine: str, output_format: str, fixture: str) -> None: + """Run one clean in this process and print a JSON result line.""" + import resource + + import freshdata as fd + + start = time.perf_counter() + # fix_dtypes uses sampled pandas heuristics, so it forces a fallback that + # materializes the frame; disabling it is part of the native-handle recipe. + result, report = fd.clean( + fixture, + engine=engine, + output_format=output_format, + strategy="conservative", + fix_dtypes=False, + return_report=True, + ) + elapsed = time.perf_counter() - start + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + if sys.platform != "darwin": # linux reports KiB, macOS bytes + peak *= 1024 + print( + json.dumps( + { + "engine": engine, + "output_format": output_format, + "materialized": report.materialized, + "peak_rss_mb": round(peak / 1e6, 1), + "seconds": round(elapsed, 2), + "result_type": type(result).__name__, + } + ) + ) + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--rows", type=int, default=2_000_000) + ap.add_argument("--fixture", help="reuse an existing parquet fixture") + ap.add_argument("--_child", nargs=3, help=argparse.SUPPRESS) + args = ap.parse_args() + + if args._child: + child(*args._child) + return + + with tempfile.TemporaryDirectory() as tmp: + fixture = args.fixture or str(Path(tmp) / "fixture.parquet") + if not args.fixture: + print(f"generating {args.rows:,}-row fixture ...", file=sys.stderr) + make_fixture(Path(fixture), args.rows) + + results = [] + for engine, fmt in SCENARIOS: + proc = subprocess.run( + [sys.executable, __file__, "--_child", engine, fmt, fixture], + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + print(f"{engine}/{fmt} FAILED:\n{proc.stderr}", file=sys.stderr) + continue + results.append(json.loads(proc.stdout.strip().splitlines()[-1])) + + print(f"\n{'engine':<8} {'output_format':<13} {'materialized':<13} " + f"{'peak RSS (MB)':>13} {'seconds':>8}") + for r in results: + print(f"{r['engine']:<8} {r['output_format']:<13} " + f"{str(r['materialized']):<13} {r['peak_rss_mb']:>13} {r['seconds']:>8}") + + +if __name__ == "__main__": + main() diff --git a/docs/backends.md b/docs/backends.md index a936de9d..110022f8 100644 --- a/docs/backends.md +++ b/docs/backends.md @@ -36,14 +36,22 @@ memory: | `"duckdb"` | `DuckDBPyRelation` (un-fetched) | **No** — you call `.fetchdf()`/`.arrow()` | | `"polars-lazy"` | `pl.LazyFrame` (un-collected) | **No** — you call `.collect()` | -Only the `"duckdb"` and `"polars-lazy"` handles are larger-than-RAM safe end to -end: nothing is fetched/collected until *you* ask. When a clean returns a native +Neither handle fetches/collects the *result* until you ask, but they are not +equal during the pipeline: the DuckDB path keeps peak memory well below the +eager equivalent, while the Polars pipeline currently collects intermediates +eagerly between stages, so `"polars-lazy"` defers only the final +materialization — peak RSS during cleaning is comparable to eager output +(measured; reproduce with `python benchmarks/bench_outofcore.py`). When a clean returns a native handle, `report.materialized` is `False` and `report.summary()` says so plainly. If the requested strategy needs the pandas decision engine (e.g. `balanced`/ `aggressive` imputation, dtype heuristics), the backend transparently falls back to pandas — that fallback is recorded in `report.fallback_events`, and the result -is materialized. Use `strategy="conservative"` (deterministic representation -repair + structural reduction) to keep the native handle. +is materialized. To keep the native handle use `strategy="conservative"` +(deterministic representation repair + structural reduction) **and** +`fix_dtypes=False` — dtype fixing relies on sampled pandas heuristics and +forces the fallback even under `conservative`. Peak-RSS evidence for the +four engine/output combinations is reproducible with +`python benchmarks/bench_outofcore.py`. **Semantic cleaning stays native too.** When `semantic_mode` is enabled on a Polars/DuckDB engine, the semantic stage runs over a *natively extracted* diff --git a/docs/limitations.md b/docs/limitations.md index 396b4ee6..c729d4a5 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -34,15 +34,18 @@ The short version: | `fd.clean(df)` (default pandas) | In-memory; the whole frame is in RAM. | | `engine="polars"` / `"duckdb"` / `"spark"`, `output_format="pandas"` (default) | Scales out **during** the pipeline (spill-to-disk), then **materializes** the result into a pandas frame. | | `output_format="duckdb"` | Returns an un-fetched `DuckDBPyRelation` — **not materialized**; you call `.fetchdf()`. | -| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — **not materialized**; you call `.collect()`. | +| `output_format="polars-lazy"` | Returns an uncollected `LazyFrame` — the **result** is not materialized (you call `.collect()`), but pipeline stages currently collect intermediates eagerly, so peak memory *during* cleaning is comparable to eager output. DuckDB is the lower-peak-memory native path today (measured: `python benchmarks/bench_outofcore.py`). | | `StreamingCleaner` / `fd.clean_timeseries(..., stream=...)` | Genuinely out-of-core: bounded micro-batches, running statistics, never concatenated. | `report.materialized` is `False` whenever a native handle is returned, and `report.summary()` says so. If a strategy needs the pandas decision engine (`balanced` / `aggressive` imputation, dtype heuristics), the native backends transparently fall back to pandas — recorded in `report.fallback_events` — and -the result is materialized. Use `strategy="conservative"` to keep the native -handle. Streaming holds memory flat by design: bounded reservoirs and counters, +the result is materialized. To keep the native handle use +`strategy="conservative"` **and** `fix_dtypes=False` — dtype fixing uses +sampled pandas heuristics and forces the fallback even under `conservative` +(measure it yourself: `python benchmarks/bench_outofcore.py`). +Streaming holds memory flat by design: bounded reservoirs and counters, a recent-window (not global) dedup, and no cross-batch concatenation. --- diff --git a/pyproject.toml b/pyproject.toml index a685ddfd..5f2119d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -346,6 +346,7 @@ ignore = [ # `import freshdata` stay dependency-free. "src/freshdata/streaming/*" = ["PLC0415"] "benchmarks/bench_streaming.py" = ["PLC0415"] +"benchmarks/bench_outofcore.py" = ["PLC0415"] "benchmarks/bench_native_semantic.py" = ["PLC0415"] "benchmarks/bench_report.py" = ["PLC0415"] "benchmarks/bench_freshcore.py" = ["E402", "I001"]