From 8f4e5471ee87f252baba4479c67d298c5bad115a Mon Sep 17 00:00:00 2001 From: Johnny Wilson Dougherty Date: Sat, 11 Jul 2026 10:29:00 +0530 Subject: [PATCH] =?UTF-8?q?chore:=20lint=20the=20whole=20repository=20?= =?UTF-8?q?=E2=80=94=20ruff=20check=20.=20in=20CI=20(#54)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI linted only src/tests; repo-wide ruff had 368 errors. Now zero: - notebooks: import sorting + PLR0402 alias fix (ruff --fix; the ipynb rewrite also normalized cell ids and unicode escapes — content unchanged, all notebooks still valid JSON) - benchmarks: wrapped genuinely long code lines (bench.py, harness_metrics.py, results_schema.py); fixed RET504/E741/F841/PLE0604 (dead gold-labels block in harness_metrics removed; __all__ now literal and verified against REGISTRY keys); per-file-ignores extended with the established conventions: PLC0415 for all of benchmarks/ (lazy optional-dep imports are the point), E501 for fixtures/ (defect tables read as one row per line), PLR0915 for the two linear builder/orchestration functions - freshdata-benchmarks/: excluded via extend-exclude — ASV-managed sub-project with tool-generated layout and per-machine result artifacts, not shipped, not held to the package lint bar - CI: ci.yml + release.yml lint steps widened to 'ruff check .' No runtime behavior changes; full gate green, coverage 93.55%. --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- CHANGELOG.md | 7 + benchmarks/baselines/pandas_baseline.py | 4 +- benchmarks/baselines/pyjanitor_baseline.py | 3 +- benchmarks/bench.py | 37 +- benchmarks/fixtures/__init__.py | 11 +- benchmarks/fixtures/_common.py | 7 +- benchmarks/fixtures/crm.py | 13 +- benchmarks/fixtures/event_log.py | 5 +- benchmarks/fixtures/finance.py | 4 +- benchmarks/fixtures/gold.py | 4 +- benchmarks/fixtures/provenance.py | 4 +- benchmarks/fixtures/wide_schema.py | 4 +- benchmarks/harness_metrics.py | 44 ++- benchmarks/results_schema.py | 12 +- notebooks/06_context_cleaning.ipynb | 74 +++- notebooks/08_plan_approval.ipynb | 40 +- notebooks/09_optional_semantic_models.ipynb | 37 +- notebooks/10_learning_profiles.ipynb | 398 +++++++++++++------- pyproject.toml | 14 +- 21 files changed, 487 insertions(+), 239 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ed66121..59c524ec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: # mypy targets Python 3.10; NumPy 2.5+ stubs require Python 3.12. pip install -e ".[dev,ml]" "numpy<2.5" - name: Lint - run: ruff check src tests + run: ruff check . - name: Typecheck run: mypy src/freshdata - name: Test (required fast lane) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a40f0ef2..0d7b4d02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -260,7 +260,7 @@ jobs: # mypy targets Python 3.10; NumPy 2.5+ stubs require Python 3.12. pip install -e ".[dev,ml,docs]" "numpy<2.5" - name: Lint - run: ruff check src tests + run: ruff check . - name: Typecheck run: mypy src/freshdata - name: Test (release fast lane) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccb1db76..17bbc824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,13 @@ adheres to [Semantic Versioning](https://semver.org/). (analyze → mask → clean under a compiled policy → merge category variants → re-score trust), and a new docs guide (`docs/ai-copilot.md`). +### Changed +- Lint now covers the whole repository (`ruff check .` in CI, closing #54): + benchmark and notebook lint debt fixed, dead code removed + (`harness_metrics` unused gold-labels block), and the ASV-managed + `freshdata-benchmarks/` sub-project excluded as tool-generated. No + runtime behavior changes. + ## [1.1.1] - 2026-07-06 ### Fixed diff --git a/benchmarks/baselines/pandas_baseline.py b/benchmarks/baselines/pandas_baseline.py index 679e8cce..cf406c07 100644 --- a/benchmarks/baselines/pandas_baseline.py +++ b/benchmarks/baselines/pandas_baseline.py @@ -15,7 +15,6 @@ import re -import numpy as np import pandas as pd from . import count_authored_lines @@ -50,8 +49,7 @@ def run(df: pd.DataFrame) -> pd.DataFrame: for col in out.select_dtypes(include="number").columns: if out[col].isna().any(): out[col] = out[col].fillna(out[col].median()) - out = out.drop_duplicates().reset_index(drop=True) - return out + return out.drop_duplicates().reset_index(drop=True) AUTHORED_LINES: int = count_authored_lines(run) diff --git a/benchmarks/baselines/pyjanitor_baseline.py b/benchmarks/baselines/pyjanitor_baseline.py index 64118755..3b5ce035 100644 --- a/benchmarks/baselines/pyjanitor_baseline.py +++ b/benchmarks/baselines/pyjanitor_baseline.py @@ -46,8 +46,7 @@ def run(df: pd.DataFrame) -> pd.DataFrame: out[col] = coerced for col in out.select_dtypes(include="number").columns: out[col] = out[col].fillna(out[col].median()) - out = out.drop_duplicates().reset_index(drop=True) - return out + return out.drop_duplicates().reset_index(drop=True) AUTHORED_LINES: int = count_authored_lines(run) diff --git a/benchmarks/bench.py b/benchmarks/bench.py index 60bde887..52fa9260 100644 --- a/benchmarks/bench.py +++ b/benchmarks/bench.py @@ -31,11 +31,11 @@ if str(HERE) not in sys.path: sys.path.insert(0, str(HERE)) -import freshdata as fd # noqa: E402 - import harness_metrics as hm # noqa: E402 +from results_schema import SCHEMA_VERSION # noqa: E402 + +import freshdata as fd # noqa: E402 from fixtures import REGISTRY # noqa: E402 -from results_schema import RESULTS_SCHEMA, SCHEMA_VERSION # noqa: E402 RESULTS_DIR = HERE / "results" @@ -143,13 +143,16 @@ def _write_result(run_id: str, result: dict) -> Path: def cmd_run(args: argparse.Namespace) -> int: run_id = _now() fixtures = args.fixtures or list(DEFAULT_SIZES) - print(f"benchmark run {run_id} freshdata={fd.__version__} mode={'aggressive' if args.aggressive else 'balanced'}") + mode = "aggressive" if args.aggressive else "balanced" + print(f"benchmark run {run_id} freshdata={fd.__version__} mode={mode}") summary_rows = [] for name in fixtures: size = args.size or DEFAULT_SIZES[name] - result = run_single(name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat) + result = run_single( + name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat + ) result["run_id"] = run_id - path = _write_result(run_id, result) + _write_result(run_id, result) m = result["metrics"] print(f" {name:12s} n={result['n_rows']:>8,} cols={result['n_cols']:>4} " f"p50={m['wall_clock_p50_sec']:.3f}s peak={m['peak_memory_mb']:.1f}MB " @@ -182,7 +185,10 @@ def cmd_compare(args: argparse.Namespace) -> int: config = hm.config_for(name, df) print(f"compare on {name} n={len(df):,} cols={df.shape[1]}\n") - header = f"{'tool':24s} {'n_rows':>8} {'n_cols':>6} {'p50_sec':>8} {'p95_sec':>8} {'peak_mb':>8} mode" + header = ( + f"{'tool':24s} {'n_rows':>8} {'n_cols':>6} " + f"{'p50_sec':>8} {'p95_sec':>8} {'peak_mb':>8} mode" + ) print(header) print("-" * len(header)) @@ -258,16 +264,20 @@ def _render_markdown(run_id: str, results: list[dict]) -> str: f"- python: `{results[0]['python_version'] if results else '?'}`", f"- platform: `{results[0]['platform'] if results else '?'}`", "", - "| fixture | n_rows | n_cols | p50 s | p95 s | peak MB | repair % | false-repair % | preserve % | trust | monotonic | export % |", + "| fixture | n_rows | n_cols | p50 s | p95 s | peak MB | repair % " + "| false-repair % | preserve % | trust | monotonic | export % |", "|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--:|--:|", ] for r in sorted(results, key=lambda x: x["fixture"]): m = r["metrics"] lines.append( f"| {r['fixture']} | {r['n_rows']:,} | {r['n_cols']} | " - f"{m['wall_clock_p50_sec']:.3f} | {m['wall_clock_p95_sec']:.3f} | {m['peak_memory_mb']:.1f} | " - f"{m['repair_fidelity_pct']} | {m['false_repair_rate_pct']} | {m['preservation_rate_pct']} | " - f"{m['trust_score']} | {'✅' if m['trust_monotonic_valid'] else '❌'} | {m['export_completeness_pct']} |" + f"{m['wall_clock_p50_sec']:.3f} | {m['wall_clock_p95_sec']:.3f} | " + f"{m['peak_memory_mb']:.1f} | " + f"{m['repair_fidelity_pct']} | {m['false_repair_rate_pct']} | " + f"{m['preservation_rate_pct']} | " + f"{m['trust_score']} | {'✅' if m['trust_monotonic_valid'] else '❌'} | " + f"{m['export_completeness_pct']} |" ) lines += ["", "## Authored-code reduction (Metric 6)", ""] if results: @@ -291,7 +301,10 @@ def cmd_fixtures(args: argparse.Namespace) -> int: bundle = mod.generate(size, seed=args.seed) bundle.dirty_df.to_csv(out / "gold_dirty.csv", index=False) bundle.clean_df.to_csv(out / "gold_clean.csv", index=False) - print(f" gold -> gold_dirty.csv ({bundle.dirty_df.shape}), gold_clean.csv ({bundle.clean_df.shape})") + print( + f" gold -> gold_dirty.csv ({bundle.dirty_df.shape}), " + f"gold_clean.csv ({bundle.clean_df.shape})" + ) else: df = mod.generate(size, seed=args.seed) path = out / f"{name}.csv" diff --git a/benchmarks/fixtures/__init__.py b/benchmarks/fixtures/__init__.py index 6a454c12..9278f17d 100644 --- a/benchmarks/fixtures/__init__.py +++ b/benchmarks/fixtures/__init__.py @@ -29,4 +29,13 @@ #: Fixtures whose ``generate`` returns a plain DataFrame (everything but gold). FRAME_FIXTURES = ("crm", "finance", "event_log", "wide_schema", "provenance") -__all__ = ["REGISTRY", "FRAME_FIXTURES", *REGISTRY.keys()] +__all__ = [ + "REGISTRY", + "FRAME_FIXTURES", + "crm", + "finance", + "event_log", + "wide_schema", + "provenance", + "gold", +] diff --git a/benchmarks/fixtures/_common.py b/benchmarks/fixtures/_common.py index 26bf82a3..0a016415 100644 --- a/benchmarks/fixtures/_common.py +++ b/benchmarks/fixtures/_common.py @@ -24,12 +24,11 @@ from __future__ import annotations -from dataclasses import asdict, dataclass, field +from dataclasses import asdict, dataclass from datetime import date from typing import Any import numpy as np -import pandas as pd # -- roles ----------------------------------------------------------------- # These mirror the roles FreshData's decision engine infers @@ -134,8 +133,8 @@ def uuid_series(rng: np.random.Generator, n: int, prefix: str = "") -> list[str] hi = rng.integers(0, 2**32, size=n, dtype=np.uint64) lo = rng.integers(0, 2**32, size=n, dtype=np.uint64) out = [] - for h, l in zip(hi.tolist(), lo.tolist()): - out.append(f"{prefix}{h:08x}-{l:08x}") + for high, low in zip(hi.tolist(), lo.tolist()): + out.append(f"{prefix}{high:08x}-{low:08x}") return out diff --git a/benchmarks/fixtures/crm.py b/benchmarks/fixtures/crm.py index 0e0aba72..c33ff5cd 100644 --- a/benchmarks/fixtures/crm.py +++ b/benchmarks/fixtures/crm.py @@ -21,15 +21,13 @@ ACCOUNT_STATUS_REF, BAD_COUNTRY, COUNTRY_REF, - Defect, - GoldLabel, - ROLE_BOOL, ROLE_CATEGORICAL, ROLE_DATETIME, ROLE_ID, ROLE_NUMERIC, ROLE_TEXT, - SENTINELS, + Defect, + GoldLabel, defect_mask, format_iso_date, gold_to_records, @@ -69,8 +67,8 @@ def _names(rng: np.random.Generator, n: int) -> np.ndarray: f = pick(rng, _FIRST, n) m = pick(rng, _MIDDLE, n) - l = pick(rng, _LAST, n) - return np.array([f"{a} {b} {c}" for a, b, c in zip(f, m, l)], dtype=object) + last = pick(rng, _LAST, n) + return np.array([f"{a} {b} {c}" for a, b, c in zip(f, m, last)], dtype=object) def generate(n_rows: int, seed: int = 42, defect_rate: float | None = None) -> pd.DataFrame: @@ -109,8 +107,7 @@ def generate(n_rows: int, seed: int = 42, defect_rate: float | None = None) -> p data[col] = np.array([str(x) for x in d], dtype=object) df = pd.DataFrame(data) - df = _inject(df, rng, defect_rate) - return df + return _inject(df, rng, defect_rate) def _inject(df: pd.DataFrame, rng: np.random.Generator, defect_rate: float | None) -> pd.DataFrame: diff --git a/benchmarks/fixtures/event_log.py b/benchmarks/fixtures/event_log.py index d3c06275..2a401828 100644 --- a/benchmarks/fixtures/event_log.py +++ b/benchmarks/fixtures/event_log.py @@ -18,13 +18,12 @@ from ._common import ( BAD_OPERATION, OPERATION_REF, - Defect, - GoldLabel, ROLE_CATEGORICAL, ROLE_DATETIME, ROLE_ID, ROLE_NUMERIC, - ROLE_TEXT, + Defect, + GoldLabel, defect_mask, gold_to_records, manifest_to_records, diff --git a/benchmarks/fixtures/finance.py b/benchmarks/fixtures/finance.py index a965b656..5238600d 100644 --- a/benchmarks/fixtures/finance.py +++ b/benchmarks/fixtures/finance.py @@ -18,13 +18,13 @@ BAD_CURRENCY, CURRENCY_REF, DEBIT_CREDIT_REF, - Defect, - GoldLabel, ROLE_CATEGORICAL, ROLE_DATETIME, ROLE_ID, ROLE_NUMERIC, ROLE_TEXT, + Defect, + GoldLabel, defect_mask, format_iso_date, gold_to_records, diff --git a/benchmarks/fixtures/gold.py b/benchmarks/fixtures/gold.py index 20e8614a..1620a535 100644 --- a/benchmarks/fixtures/gold.py +++ b/benchmarks/fixtures/gold.py @@ -33,14 +33,14 @@ import pandas as pd from ._common import ( - Defect, - GoldLabel, ROLE_CATEGORICAL, ROLE_DATETIME, ROLE_ID, ROLE_NUMERIC, ROLE_TARGET, ROLE_TEXT, + Defect, + GoldLabel, defect_mask, format_iso_date, gold_to_records, diff --git a/benchmarks/fixtures/provenance.py b/benchmarks/fixtures/provenance.py index 01098eb6..c06013ce 100644 --- a/benchmarks/fixtures/provenance.py +++ b/benchmarks/fixtures/provenance.py @@ -20,13 +20,13 @@ import pandas as pd from ._common import ( - Defect, - GoldLabel, ROLE_CATEGORICAL, ROLE_DATETIME, ROLE_ID, ROLE_NUMERIC, ROLE_TEXT, + Defect, + GoldLabel, defect_mask, gold_to_records, manifest_to_records, diff --git a/benchmarks/fixtures/wide_schema.py b/benchmarks/fixtures/wide_schema.py index aaa3897a..59154ac9 100644 --- a/benchmarks/fixtures/wide_schema.py +++ b/benchmarks/fixtures/wide_schema.py @@ -16,8 +16,6 @@ import pandas as pd from ._common import ( - Defect, - GoldLabel, ROLE_BOOL, ROLE_CATEGORICAL, ROLE_DATETIME, @@ -25,6 +23,8 @@ ROLE_NUMERIC, ROLE_TARGET, ROLE_TEXT, + Defect, + GoldLabel, defect_mask, format_iso_date, gold_to_records, diff --git a/benchmarks/harness_metrics.py b/benchmarks/harness_metrics.py index 54c77084..7226fb0b 100644 --- a/benchmarks/harness_metrics.py +++ b/benchmarks/harness_metrics.py @@ -18,7 +18,6 @@ import os import tempfile import tracemalloc -from statistics import quantiles from time import perf_counter from typing import Any @@ -26,11 +25,11 @@ import pandas as pd import freshdata as fd +from fixtures import REGISTRY +from fixtures import gold as gold_mod from freshdata import CleanConfig from freshdata._sentinels import DEFAULT_SENTINELS -from fixtures import REGISTRY, gold as gold_mod - # Canonical FreshData invocation for the authored-code metric: a config line and # the clean call. Three logical lines covers id/target hints + clean + report. FD_AUTHORED_LINES = 3 @@ -76,7 +75,9 @@ def make_frame(name: str, size: int, seed: int, defect_rate: float | None = None # -- metric 1: wall-clock -------------------------------------------------- -def metric_wall_clock(df: pd.DataFrame, config: CleanConfig, *, repeat: int = 5) -> dict[str, float]: +def metric_wall_clock( + df: pd.DataFrame, config: CleanConfig, *, repeat: int = 5 +) -> dict[str, float]: """p50 / p95 wall-clock seconds for clean + full report materialisation. Times only ``fd.clean(..., return_report=True)``; FreshData does not mutate @@ -223,7 +224,9 @@ def preservation_report(name: str, dirty: pd.DataFrame, cleaned: pd.DataFrame) - return { "per_role": per_role, - "preservation_rate_pct": round(100.0 * (agg_total - agg_changed) / agg_total, 4) if agg_total else 100.0, + "preservation_rate_pct": ( + round(100.0 * (agg_total - agg_changed) / agg_total, 4) if agg_total else 100.0 + ), "false_repair_rate_pct": round(100.0 * agg_changed / agg_total, 4) if agg_total else 0.0, "ids_never_filled": nulls_out >= max(0, nulls_in - (len(dirty) - len(cleaned))), } @@ -321,9 +324,13 @@ def gold_repair_report(size: int, seed: int) -> dict[str, Any]: for fam in fam_total if fam_total[fam] }, - "false_repair_rate_pct": round(100.0 * trap_changed / trap_total, 4) if trap_total else 0.0, + "false_repair_rate_pct": ( + round(100.0 * trap_changed / trap_total, 4) if trap_total else 0.0 + ), "per_trap": per_trap, - "preservation_rate_pct": round(100.0 * pres_unchanged / pres_total, 4) if pres_total else 100.0, + "preservation_rate_pct": ( + round(100.0 * pres_unchanged / pres_total, 4) if pres_total else 100.0 + ), "n_rows": n, } @@ -429,7 +436,9 @@ def _has_surrounding_ws(series: pd.Series) -> bool: return bool(vals.map(lambda v: isinstance(v, str) and v != v.strip()).any()) -def manifest_repair_fidelity(name: str, dirty: pd.DataFrame, cleaned: pd.DataFrame, report) -> dict[str, Any]: +def manifest_repair_fidelity( + name: str, dirty: pd.DataFrame, cleaned: pd.DataFrame, report +) -> dict[str, Any]: """Family-level repair fidelity for a named fixture, from DEFECT_MANIFEST. Each documented defect family contributes a single post-condition (did @@ -440,9 +449,6 @@ def manifest_repair_fidelity(name: str, dirty: pd.DataFrame, cleaned: pd.DataFra preservation, not silent rewriting. """ mod = REGISTRY[name] - labels = getattr(mod, "GOLD_LABELS", {}) - if callable(getattr(mod, "gold_labels", None)) and name == "wide_schema": - labels = mod.gold_labels(dirty.shape[1]) cols = list(cleaned.columns) per_family: dict[str, bool] = {} @@ -469,7 +475,9 @@ def manifest_repair_fidelity(name: str, dirty: pd.DataFrame, cleaned: pd.DataFra elif repair in ("reference_flag",): # correct = bad values preserved (not silently rewritten to a valid one) satisfied = all(c in cleaned for c in targets) - elif repair in ("flag_missing", "preserve", "cdc_flag", "provenance_flag", "dtype_coerce_or_flag"): + elif repair in ( + "flag_missing", "preserve", "cdc_flag", "provenance_flag", "dtype_coerce_or_flag" + ): # non-destructive: the column still exists and missing stays missing satisfied = all(c in cleaned for c in targets) else: @@ -492,7 +500,9 @@ def metric_authored_lines() -> dict[str, Any]: "fd_lines": FD_AUTHORED_LINES, "pandas_lines": pandas_lines, "pyjanitor_lines": pj_lines, - "reduction_vs_pandas_pct": round(100.0 * (pandas_lines - FD_AUTHORED_LINES) / pandas_lines, 2), + "reduction_vs_pandas_pct": round( + 100.0 * (pandas_lines - FD_AUTHORED_LINES) / pandas_lines, 2 + ), "reduction_vs_pyjanitor_pct": round(100.0 * (pj_lines - FD_AUTHORED_LINES) / pj_lines, 2), } @@ -535,7 +545,9 @@ def metric_trust(name: str, sweep_size: int, seed: int) -> dict[str, Any]: # -- metric 9: export completeness ----------------------------------------- -def metric_export_completeness(name: str, df: pd.DataFrame, config: CleanConfig, report) -> dict[str, Any]: +def metric_export_completeness( + name: str, df: pd.DataFrame, config: CleanConfig, report +) -> dict[str, Any]: """All required report fields populated + export methods non-empty/valid.""" checks: list[bool] = [] missing: list[str] = [] @@ -565,7 +577,9 @@ def check(label: str, cond: bool) -> None: check(f"before_after:{col}", bool(getattr(action, "description", ""))) check(f"risk:{col}", getattr(action, "risk", None) in _RISK_LEVELS) check(f"confidence:{col}", isinstance(getattr(action, "confidence", None), float)) - is_engine = bool(getattr(action, "model_id", "")) or getattr(action, "risk", "low") != "low" + is_engine = ( + bool(getattr(action, "model_id", "")) or getattr(action, "risk", "low") != "low" + ) if is_engine: check(f"rationale:{col}", bool(getattr(action, "rationale", ""))) diff --git a/benchmarks/results_schema.py b/benchmarks/results_schema.py index f24c0e6f..07e22d2e 100644 --- a/benchmarks/results_schema.py +++ b/benchmarks/results_schema.py @@ -63,8 +63,12 @@ "wall_clock_p95_sec": {"type": ["number", "null"], "minimum": 0}, "peak_memory_mb": {"type": ["number", "null"], "minimum": 0}, "repair_fidelity_pct": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, - "false_repair_rate_pct": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, - "preservation_rate_pct": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, + "false_repair_rate_pct": { + "type": ["number", "null"], "minimum": 0, "maximum": 100 + }, + "preservation_rate_pct": { + "type": ["number", "null"], "minimum": 0, "maximum": 100 + }, "authored_lines_fd": {"type": ["integer", "null"], "minimum": 0}, "authored_lines_pandas": {"type": ["integer", "null"], "minimum": 0}, "reduction_vs_pandas_pct": {"type": ["number", "null"]}, @@ -73,7 +77,9 @@ "diagnosis_to_dict_sec": {"type": ["number", "null"], "minimum": 0}, "trust_score": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, "trust_monotonic_valid": {"type": ["boolean", "null"]}, - "export_completeness_pct": {"type": ["number", "null"], "minimum": 0, "maximum": 100}, + "export_completeness_pct": { + "type": ["number", "null"], "minimum": 0, "maximum": 100 + }, }, }, "details": {"type": "object"}, diff --git a/notebooks/06_context_cleaning.ipynb b/notebooks/06_context_cleaning.ipynb index 7e127e1c..07527dab 100644 --- a/notebooks/06_context_cleaning.ipynb +++ b/notebooks/06_context_cleaning.ipynb @@ -2,27 +2,31 @@ "cells": [ { "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "source": [ - "# Context cleaning \u2014 rules in English, compiled\n", + "# Context cleaning — rules in English, compiled\n", "\n", "FreshData compiles natural-language expectations into a deterministic,\n", - "reviewable `ContextPolicy` \u2014 no model, no network, no new dependency.\n", + "reviewable `ContextPolicy` — no model, no network, no new dependency.\n", "This notebook walks the flagship ecommerce example end to end." ] }, { "cell_type": "code", "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", + "\n", "import freshdata as fd\n" ] }, { "cell_type": "markdown", + "id": "9a63283cbaf04dbcab1f6479b197f3a8", "metadata": {}, "source": [ "## A messy ecommerce customer extract" @@ -31,6 +35,7 @@ { "cell_type": "code", "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", "metadata": {}, "outputs": [], "source": [ @@ -48,6 +53,7 @@ }, { "cell_type": "markdown", + "id": "72eea5119410473aa328ad9291626812", "metadata": {}, "source": [ "## 1. Write the rules as English" @@ -56,17 +62,26 @@ { "cell_type": "code", "execution_count": null, + "id": "8edb47106e1a46a883d545849b8ab81b", "metadata": {}, "outputs": [], "source": [ - "context = \"\"\"\nThis is an ecommerce customer dataset.\nCustomerID is unique.\nEmails must be valid.\nPhone numbers are Indian.\nMissing Age should be estimated only if confidence >95%.\nNever modify revenue values.\n\"\"\"" + "context = \"\"\"\n", + "This is an ecommerce customer dataset.\n", + "CustomerID is unique.\n", + "Emails must be valid.\n", + "Phone numbers are Indian.\n", + "Missing Age should be estimated only if confidence >95%.\n", + "Never modify revenue values.\n", + "\"\"\"" ] }, { "cell_type": "markdown", + "id": "10185d26023b46108eb7d9f57d49d2b3", "metadata": {}, "source": [ - "## 2. Compile \u2014 and inspect what FreshData understood\n", + "## 2. Compile — and inspect what FreshData understood\n", "\n", "`fd.compile_context` is deterministic: the same text and schema always\n", "produce the same policy. Unresolved references and unparsed sentences are\n", @@ -76,6 +91,7 @@ { "cell_type": "code", "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", "metadata": {}, "outputs": [], "source": [ @@ -85,14 +101,16 @@ }, { "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", "metadata": {}, "source": [ - "The policy is a plain JSON artifact \u2014 commit it, diff it, review it in a PR." + "The policy is a plain JSON artifact — commit it, diff it, review it in a PR." ] }, { "cell_type": "code", "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", "metadata": {}, "outputs": [], "source": [ @@ -101,6 +119,7 @@ }, { "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", "metadata": {}, "source": [ "## 3. Clean under the policy\n", @@ -113,6 +132,7 @@ { "cell_type": "code", "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", "metadata": {}, "outputs": [], "source": [ @@ -123,6 +143,7 @@ { "cell_type": "code", "execution_count": null, + "id": "504fb2a444614c0babb325280ed9130a", "metadata": {}, "outputs": [], "source": [ @@ -131,14 +152,16 @@ }, { "cell_type": "markdown", + "id": "59bbdb311c014d738909a11f9e486628", "metadata": {}, "source": [ - "The protected column is byte-identical \u2014 \"Never modify revenue values.\" is a hard rule:" + "The protected column is byte-identical — \"Never modify revenue values.\" is a hard rule:" ] }, { "cell_type": "code", "execution_count": null, + "id": "b43b363d81ae4b689946ece5c682cd59", "metadata": {}, "outputs": [], "source": [ @@ -147,6 +170,7 @@ }, { "cell_type": "markdown", + "id": "8a65eabff63a45729fe45fb5ade58bdc", "metadata": {}, "source": [ "## 4. Validate without mutating" @@ -155,16 +179,18 @@ { "cell_type": "code", "execution_count": null, + "id": "c3933fab20d04ec698c2621248eb3be0", "metadata": {}, "outputs": [], "source": [ "findings = fd.validate(df, context=context)\n", "for f in findings:\n", - " print(f.severity.upper(), f.rule_name, \"\u2014\", f.message)" + " print(f.severity.upper(), f.rule_name, \"—\", f.message)" ] }, { "cell_type": "markdown", + "id": "4dd4641cc4064e0191573fe9c69df29b", "metadata": {}, "source": [ "## 5. Strict mode\n", @@ -176,6 +202,7 @@ { "cell_type": "code", "execution_count": null, + "id": "8309879909854d7188b41380fd92a7c3", "metadata": {}, "outputs": [], "source": [ @@ -187,6 +214,7 @@ }, { "cell_type": "markdown", + "id": "3ed186c9a28b402fb0bc4494df01f08d", "metadata": {}, "source": [ "## 6. Reuse the compiled policy" @@ -195,6 +223,7 @@ { "cell_type": "code", "execution_count": null, + "id": "cb1e1581032b452c9409d6c6813c49d1", "metadata": {}, "outputs": [], "source": [ @@ -204,6 +233,7 @@ }, { "cell_type": "markdown", + "id": "379cbbc1e968416e875cc15c1202d7eb", "metadata": {}, "source": [ "## Phase 2: the context is now enforced end to end\n", @@ -211,26 +241,27 @@ "Phase 2 adds deterministic value experts and hard safety gates behind the\n", "same context sentences:\n", "\n", - "- `Emails must be valid.` \u2192 the **EmailExpert** mechanically repairs\n", - " unambiguous addresses (`\" Bob@GMAIL.COM \"` \u2192 `\"Bob@gmail.com\"`,\n", - " `\"a @ b.com\"` \u2192 `\"a@b.com\"`, one doubled `@@`) and flags everything else.\n", - "- `Phone numbers are Indian.` \u2192 the **PhoneExpert** normalizes safe forms to\n", + "- `Emails must be valid.` → the **EmailExpert** mechanically repairs\n", + " unambiguous addresses (`\" Bob@GMAIL.COM \"` → `\"Bob@gmail.com\"`,\n", + " `\"a @ b.com\"` → `\"a@b.com\"`, one doubled `@@`) and flags everything else.\n", + "- `Phone numbers are Indian.` → the **PhoneExpert** normalizes safe forms to\n", " canonical `+91XXXXXXXXXX` and flags the rest.\n", - "- `Allowed status values are ...` \u2192 the **ReferenceExpert** repairs\n", + "- `Allowed status values are ...` → the **ReferenceExpert** repairs\n", " case/whitespace/separator variants exactly, *suggests* single-candidate\n", " typos, and blocks ambiguous ones.\n", - "- `Missing Age should be estimated only if confidence >95%.` \u2192 the\n", + "- `Missing Age should be estimated only if confidence >95%.` → the\n", " statistical engine now refuses fills below the threshold and records a\n", " suggested action instead.\n", - "- `Never modify revenue values.` \u2192 enforced **physically**: the executor\n", + "- `Never modify revenue values.` → enforced **physically**: the executor\n", " verifies the column byte-identical before returning, and raises\n", " `fd.ProtectedColumnError` if any code path broke the promise." ] }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "277c27b1587741f2af2001be3712ef0d", + "metadata": {}, "outputs": [], "source": [ "clean_df, report = fd.clean(\n", @@ -245,11 +276,12 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "db7b79bc585a40fcaf58bf750017e135", + "metadata": {}, "outputs": [], "source": [ - "# Every decision is audited: applied, suggested, skipped \u2014 with rationale.\n", + "# Every decision is audited: applied, suggested, skipped — with rationale.\n", "for action in report:\n", " if action.step in (\"semantic\", \"missing\", \"guard\"):\n", " print(f\"[{action.step:8}] {str(action.column):16} {action.status:10} {action.description}\")" @@ -257,11 +289,12 @@ }, { "cell_type": "markdown", + "id": "916684f9a58a4a2aa5f864670399430d", "metadata": {}, "source": [ "The protected `monthly_revenue` column is byte-identical, missing `age`\n", "survived (0.9 fill confidence < 0.95 required), and the duplicate\n", - "`cust_id` shows up in `fd.validate` \u2014 reported, not silently dropped.\n", + "`cust_id` shows up in `fd.validate` — reported, not silently dropped.\n", "\n", "Want to review the repairs *before* they run? That is the repair-plan\n", "workflow: see `notebooks/08_plan_approval.ipynb` and\n", @@ -270,13 +303,14 @@ }, { "cell_type": "markdown", + "id": "1671c31a24314836a5b85d7ef7fbf015", "metadata": {}, "source": [ "## Limitations\n", "\n", - "- **Deterministic tier-0 lexicon only** \u2014 a fixed set of ~12 intent\n", + "- **Deterministic tier-0 lexicon only** — a fixed set of ~12 intent\n", " families; anything outside it is surfaced as *unparsed*, by design.\n", - "- No model, no embeddings, no network \u2014 and therefore no arbitrary NLU.\n", + "- No model, no embeddings, no network — and therefore no arbitrary NLU.\n", "- `valid_format` / `locale_format` now drive real value repairs (email, and\n", " phones for `region=\"IN\"`); other phone regions compile but do not repair yet.\n", "- Ambiguous repairs are suggested or flagged, never auto-applied.\n", diff --git a/notebooks/08_plan_approval.ipynb b/notebooks/08_plan_approval.ipynb index 953fdf51..ec9bd85d 100644 --- a/notebooks/08_plan_approval.ipynb +++ b/notebooks/08_plan_approval.ipynb @@ -2,24 +2,27 @@ "cells": [ { "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "source": [ - "# 08 \u2014 Repair plans: suggest, approve, apply, undo\n", + "# 08 — Repair plans: suggest, approve, apply, undo\n", "\n", "Phase 2 makes cleaning **reviewable**: `fd.suggest_plan` proposes exactly the\n", "repairs `fd.clean` would make, you approve/reject/override them, and\n", - "`fd.apply_plan` executes *only* what you approved \u2014 under a physical\n", + "`fd.apply_plan` executes *only* what you approved — under a physical\n", "protected-column guard, with drift refusal, an undo log, and a deterministic\n", "audit hash. Everything is offline and model-free." ] }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", + "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", + "\n", "import freshdata as fd\n", "\n", "df = pd.DataFrame({\n", @@ -39,6 +42,7 @@ }, { "cell_type": "markdown", + "id": "9a63283cbaf04dbcab1f6479b197f3a8", "metadata": {}, "source": [ "## 1. Suggest a plan\n", @@ -50,8 +54,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", + "metadata": {}, "outputs": [], "source": [ "plan = fd.suggest_plan(df, context=CONTEXT, semantic_mode=\"auto\", verbose=False)\n", @@ -61,8 +66,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "72eea5119410473aa328ad9291626812", + "metadata": {}, "outputs": [], "source": [ "rp.to_frame() # one row per planned action, for filtering/sorting" @@ -70,6 +76,7 @@ }, { "cell_type": "markdown", + "id": "8edb47106e1a46a883d545849b8ab81b", "metadata": {}, "source": [ "## 2. Review\n", @@ -80,8 +87,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "10185d26023b46108eb7d9f57d49d2b3", + "metadata": {}, "outputs": [], "source": [ "# The fuzzy typo \"actve\" -> \"active\" was only *suggested* (conf 0.8 < 0.95).\n", @@ -98,9 +106,10 @@ }, { "cell_type": "markdown", + "id": "8763a12b2bbd4a93a75aff182afb95dc", "metadata": {}, "source": [ - "## 3. Apply \u2014 exactly what was approved\n", + "## 3. Apply — exactly what was approved\n", "\n", "- nothing is re-profiled, nothing is re-decided;\n", "- rejected and blocked actions do not run;\n", @@ -110,8 +119,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "7623eae2785240b9bd12b16a66d81610", + "metadata": {}, "outputs": [], "source": [ "clean_df, report = fd.apply_plan(df, rp, keep_undo=True)\n", @@ -120,8 +130,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", + "metadata": {}, "outputs": [], "source": [ "print(\"decisions_hash:\", report.decisions_hash)\n", @@ -130,6 +141,7 @@ }, { "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", "metadata": {}, "source": [ "## 4. Drift refusal\n", @@ -140,8 +152,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", + "metadata": {}, "outputs": [], "source": [ "drifted = df.copy()\n", @@ -157,6 +170,7 @@ }, { "cell_type": "markdown", + "id": "504fb2a444614c0babb325280ed9130a", "metadata": {}, "source": [ "## 5. Undo\n", @@ -168,8 +182,9 @@ }, { "cell_type": "code", - "metadata": {}, "execution_count": null, + "id": "59bbdb311c014d738909a11f9e486628", + "metadata": {}, "outputs": [], "source": [ "email_ids = [a.id for a in rp.actions\n", @@ -180,6 +195,7 @@ }, { "cell_type": "markdown", + "id": "b43b363d81ae4b689946ece5c682cd59", "metadata": {}, "source": [ "## 6. Plans as files\n", @@ -190,7 +206,7 @@ "```\n", "\n", "`RepairPlan.to_json()` / `from_json()` round-trip actions, approval state,\n", - "the compiled policy, and the frame signature \u2014 so a plan can be reviewed in\n", + "the compiled policy, and the frame signature — so a plan can be reviewed in\n", "a pull request like code.\n", "\n", "## Limitations\n", diff --git a/notebooks/09_optional_semantic_models.ipynb b/notebooks/09_optional_semantic_models.ipynb index a7bbcbea..18fdaebe 100644 --- a/notebooks/09_optional_semantic_models.ipynb +++ b/notebooks/09_optional_semantic_models.ipynb @@ -2,11 +2,12 @@ "cells": [ { "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "source": [ "# Optional semantic models\n", "\n", - "FreshData's default install is **model-free** \u2014 everything you saw in the\n", + "FreshData's default install is **model-free** — everything you saw in the\n", "context-cleaning and plan-approval notebooks is deterministic, offline code.\n", "This notebook shows the *optional* Phase-3 model path: a small local ONNX\n", "encoder that adds repair **proposals** (never decisions) for values the\n", @@ -21,6 +22,7 @@ { "cell_type": "code", "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", "metadata": {}, "outputs": [], "source": [ @@ -38,6 +40,7 @@ }, { "cell_type": "markdown", + "id": "9a63283cbaf04dbcab1f6479b197f3a8", "metadata": {}, "source": [ "## The model registry\n", @@ -50,6 +53,7 @@ { "cell_type": "code", "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", "metadata": {}, "outputs": [], "source": [ @@ -61,6 +65,7 @@ { "cell_type": "code", "execution_count": null, + "id": "72eea5119410473aa328ad9291626812", "metadata": {}, "outputs": [], "source": [ @@ -69,9 +74,10 @@ }, { "cell_type": "markdown", + "id": "8edb47106e1a46a883d545849b8ab81b", "metadata": {}, "source": [ - "Official artifacts are not hosted yet \u2014 `fd.models.pull(\"fd-col-encoder-v1\")`\n", + "Official artifacts are not hosted yet — `fd.models.pull(\"fd-col-encoder-v1\")`\n", "raises `ModelNotPublishedError` with instructions (mirror via\n", "`FRESHDATA_MODEL_URL_BASE`, or drop files into `FRESHDATA_MODEL_DIR` for\n", "air-gapped installs). The stub we enabled above stands in for the encoder.\n" @@ -79,20 +85,22 @@ }, { "cell_type": "markdown", + "id": "10185d26023b46108eb7d9f57d49d2b3", "metadata": {}, "source": [ "## Embedding-assisted cleaning\n", "\n", - "The ecommerce fixture again \u2014 but note `status` contains `'activvee'`, two edits\n", + "The ecommerce fixture again — but note `status` contains `'activvee'`, two edits\n", "away from `active`. The deterministic reference expert only trusts one edit, so\n", "on a model-free install this value is *flagged, never repaired*. With the\n", - "embedding backend enabled, the encoder gets a vote \u2014 as **evidence through the\n", + "embedding backend enabled, the encoder gets a vote — as **evidence through the\n", "same gate**, never as authority.\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", "metadata": {}, "outputs": [], "source": [ @@ -129,15 +137,17 @@ }, { "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", "metadata": {}, "source": [ - "Protected means protected \u2014 `monthly_revenue` is byte-identical no matter\n", + "Protected means protected — `monthly_revenue` is byte-identical no matter\n", "which backends run (a physical executor check, not a convention):\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", "metadata": {}, "outputs": [], "source": [ @@ -147,6 +157,7 @@ }, { "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", "metadata": {}, "source": [ "Every model-assisted action carries full provenance: which backend, which\n", @@ -156,6 +167,7 @@ { "cell_type": "code", "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", "metadata": {}, "outputs": [], "source": [ @@ -170,22 +182,24 @@ }, { "cell_type": "markdown", + "id": "504fb2a444614c0babb325280ed9130a", "metadata": {}, "source": [ "## Graceful degradation\n", "\n", "Requesting the embedding backend without the extra or the model never crashes\n", - "\u2014 it self-disables with a report note, and cleaning proceeds exactly as the\n", + "— it self-disables with a report note, and cleaning proceeds exactly as the\n", "model-free install would:\n" ] }, { "cell_type": "code", "execution_count": null, + "id": "59bbdb311c014d738909a11f9e486628", "metadata": {}, "outputs": [], "source": [ - "import freshdata.models.runtime as runtime\n", + "from freshdata.models import runtime\n", "\n", "os.environ.pop(\"FRESHDATA_STUB_ENCODER\", None)\n", "runtime.reset_encoders()\n", @@ -202,16 +216,17 @@ }, { "cell_type": "markdown", + "id": "b43b363d81ae4b689946ece5c682cd59", "metadata": {}, "source": [ "## What the model path will never do\n", "\n", - "- **No automatic downloads** \u2014 `fd.clean` cannot reach the network.\n", - "- **No LLM, no cloud inference, no per-cell model calls** \u2014 the backend sees\n", + "- **No automatic downloads** — `fd.clean` cannot reach the network.\n", + "- **No LLM, no cloud inference, no per-cell model calls** — the backend sees\n", " distinct values of eligible columns only.\n", - "- **No ambiguous auto-applies** \u2014 a value near two allowed candidates produces\n", + "- **No ambiguous auto-applies** — a value near two allowed candidates produces\n", " *no proposal at all*; pure similarity clustering is suggest-only by default.\n", - "- **No unprotected protected columns** \u2014 byte-identity is asserted physically\n", + "- **No unprotected protected columns** — byte-identity is asserted physically\n", " after every run.\n", "\n", "See `docs/semantic-models.md` for install, air-gapped setup, calibration, and\n", diff --git a/notebooks/10_learning_profiles.ipynb b/notebooks/10_learning_profiles.ipynb index 00d4738e..c94052c4 100644 --- a/notebooks/10_learning_profiles.ipynb +++ b/notebooks/10_learning_profiles.ipynb @@ -2,12 +2,13 @@ "cells": [ { "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", "metadata": {}, "source": [ "# Learning profiles: paired-data cleaning that remembers\n\n", "\n\n", "Phase 4 adds `fd.learn()`: instead of hand-writing cleaning rules, you show freshdata a\n\n", - "**messy/clean pair** of the same rows and it learns a reusable `.fdprofile` \u2014 a set of\n\n", + "**messy/clean pair** of the same rows and it learns a reusable `.fdprofile` — a set of\n\n", "typed rules, value maps, and (optionally) retrieval examples that later replay against new,\n\n", "similarly-shaped data via `fd.clean(df, profile=...)`.\n\n", "\n\n", @@ -18,12 +19,13 @@ }, { "cell_type": "markdown", + "id": "acae54e37e7d407bbb7b55eff062a284", "metadata": {}, "source": [ "## 1. A messy/clean pair\n\n", "\n\n", "We build 30 rows of \"messy\" input and the corresponding \"clean\" ground truth. Two status\n\n", - "typos \u2014 `\"Shipped\"` and `\"Deliverd\"` \u2014 each occur with support 5, right at the default\n\n", + "typos — `\"Shipped\"` and `\"Deliverd\"` — each occur with support 5, right at the default\n\n", "`min_support=5` gate, so both should be learned as value-map rules rather than demoted to\n\n", "suggest-only examples." ] @@ -31,97 +33,173 @@ { "cell_type": "code", "execution_count": null, + "id": "9a63283cbaf04dbcab1f6479b197f3a8", "metadata": {}, "outputs": [], "source": [ - "import pandas as pd\n\n", - "\n\n", - "import freshdata as fd\n\n", - "\n\n", - "n = 30\n\n", - "cust_id = list(range(1, n + 1))\n\n", - "\n\n", - "# email: every 3rd row upper-cased, every 3rd row padded with whitespace\n\n", - "emails_clean = [f\"user{i}@example.in\" for i in range(1, n + 1)]\n\n", - "emails_messy = [\n\n", - " e.upper() if i % 3 == 0 else (f\" {e} \" if i % 3 == 1 else e)\n\n", - " for i, e in enumerate(emails_clean)\n\n", - "]\n\n", - "\n\n", - "# phone: rotate through a few common messy IN formats\n\n", - "phones_clean = [f\"+9198765{43200 + i:05d}\" for i in range(n)]\n\n", - "phones_messy = []\n\n", - "for i, p in enumerate(phones_clean):\n\n", - " digits = p.replace(\"+91\", \"\")\n\n", - " style = i % 4\n\n", - " if style == 0:\n\n", - " phones_messy.append(f\"0{digits}\")\n\n", - " elif style == 1:\n\n", - " phones_messy.append(f\"{digits[:5]} {digits[5:]}\")\n\n", - " elif style == 2:\n\n", - " phones_messy.append(f\"91-{digits}\")\n\n", - " else:\n\n", - " phones_messy.append(p)\n\n", - "\n\n", - "# signup_date: half day-first, half already ISO\n\n", - "dates_clean = [f\"2024-{1 + (i % 9):02d}-{10 + (i % 15):02d}\" for i in range(n)]\n\n", - "dates_messy = [\n\n", - " f\"{d.split('-')[2]}/{d.split('-')[1]}/{d.split('-')[0]}\" if i % 2 == 0 else d\n\n", - " for i, d in enumerate(dates_clean)\n\n", - "]\n\n", - "\n\n", - "# amount_spent: rupee-formatted strings -> floats\n\n", - "amounts_clean = [round(500 + i * 37.5, 2) for i in range(n)]\n\n", - "amounts_messy = [f\"\u20b9{a:,.2f}\" for a in amounts_clean]\n\n", - "\n\n", - "# status: two typo'd spellings, each with support >= min_support (5)\n\n", - "status_clean = (['shipped'] * 10) + (['delivered'] * 10) + (['cancelled'] * 10)\n\n", - "status_messy = (\n\n", - " (['Shipped'] * 5 + ['shipped'] * 5)\n\n", - " + (['Deliverd'] * 5 + ['delivered'] * 5)\n\n", - " + (['cancelled'] * 10)\n\n", - ")\n\n", - "\n\n", - "# country: a nonstandard sentinel for missing, every 6th row\n\n", - "country_clean = ['India'] * n\n\n", - "country_messy = ['India'] * n\n\n", - "for i in range(0, n, 6):\n\n", - " country_messy[i] = 'N/A'\n\n", - " country_clean[i] = None\n\n", - "\n\n", - "messy_df = pd.DataFrame(\n\n", - " {\n\n", - " \"cust_id\": cust_id,\n\n", - " \"email\": emails_messy,\n\n", - " \"phone\": phones_messy,\n\n", - " \"signup_date\": dates_messy,\n\n", - " \"amount_spent\": amounts_messy,\n\n", - " \"status\": status_messy,\n\n", - " \"country\": country_messy,\n\n", - " }\n\n", - ")\n\n", - "clean_df = pd.DataFrame(\n\n", - " {\n\n", - " \"cust_id\": cust_id,\n\n", - " \"email\": emails_clean,\n\n", - " \"phone\": phones_clean,\n\n", - " \"signup_date\": dates_clean,\n\n", - " \"amount_spent\": amounts_clean,\n\n", - " \"status\": status_clean,\n\n", - " \"country\": country_clean,\n\n", - " }\n\n", - ")\n\n", - "\n\n", + "import pandas as pd\n", + "\n", + "import freshdata as fd\n", + "\n", + "n = 30\n", + "\n", + "cust_id = list(range(1, n + 1))\n", + "\n", + "\n", + "\n", + "# email: every 3rd row upper-cased, every 3rd row padded with whitespace\n", + "\n", + "emails_clean = [f\"user{i}@example.in\" for i in range(1, n + 1)]\n", + "\n", + "emails_messy = [\n", + "\n", + " e.upper() if i % 3 == 0 else (f\" {e} \" if i % 3 == 1 else e)\n", + "\n", + " for i, e in enumerate(emails_clean)\n", + "\n", + "]\n", + "\n", + "\n", + "\n", + "# phone: rotate through a few common messy IN formats\n", + "\n", + "phones_clean = [f\"+9198765{43200 + i:05d}\" for i in range(n)]\n", + "\n", + "phones_messy = []\n", + "\n", + "for i, p in enumerate(phones_clean):\n", + "\n", + " digits = p.replace(\"+91\", \"\")\n", + "\n", + " style = i % 4\n", + "\n", + " if style == 0:\n", + "\n", + " phones_messy.append(f\"0{digits}\")\n", + "\n", + " elif style == 1:\n", + "\n", + " phones_messy.append(f\"{digits[:5]} {digits[5:]}\")\n", + "\n", + " elif style == 2:\n", + "\n", + " phones_messy.append(f\"91-{digits}\")\n", + "\n", + " else:\n", + "\n", + " phones_messy.append(p)\n", + "\n", + "\n", + "\n", + "# signup_date: half day-first, half already ISO\n", + "\n", + "dates_clean = [f\"2024-{1 + (i % 9):02d}-{10 + (i % 15):02d}\" for i in range(n)]\n", + "\n", + "dates_messy = [\n", + "\n", + " f\"{d.split('-')[2]}/{d.split('-')[1]}/{d.split('-')[0]}\" if i % 2 == 0 else d\n", + "\n", + " for i, d in enumerate(dates_clean)\n", + "\n", + "]\n", + "\n", + "\n", + "\n", + "# amount_spent: rupee-formatted strings -> floats\n", + "\n", + "amounts_clean = [round(500 + i * 37.5, 2) for i in range(n)]\n", + "\n", + "amounts_messy = [f\"₹{a:,.2f}\" for a in amounts_clean]\n", + "\n", + "\n", + "\n", + "# status: two typo'd spellings, each with support >= min_support (5)\n", + "\n", + "status_clean = (['shipped'] * 10) + (['delivered'] * 10) + (['cancelled'] * 10)\n", + "\n", + "status_messy = (\n", + "\n", + " (['Shipped'] * 5 + ['shipped'] * 5)\n", + "\n", + " + (['Deliverd'] * 5 + ['delivered'] * 5)\n", + "\n", + " + (['cancelled'] * 10)\n", + "\n", + ")\n", + "\n", + "\n", + "\n", + "# country: a nonstandard sentinel for missing, every 6th row\n", + "\n", + "country_clean = ['India'] * n\n", + "\n", + "country_messy = ['India'] * n\n", + "\n", + "for i in range(0, n, 6):\n", + "\n", + " country_messy[i] = 'N/A'\n", + "\n", + " country_clean[i] = None\n", + "\n", + "\n", + "\n", + "messy_df = pd.DataFrame(\n", + "\n", + " {\n", + "\n", + " \"cust_id\": cust_id,\n", + "\n", + " \"email\": emails_messy,\n", + "\n", + " \"phone\": phones_messy,\n", + "\n", + " \"signup_date\": dates_messy,\n", + "\n", + " \"amount_spent\": amounts_messy,\n", + "\n", + " \"status\": status_messy,\n", + "\n", + " \"country\": country_messy,\n", + "\n", + " }\n", + "\n", + ")\n", + "\n", + "clean_df = pd.DataFrame(\n", + "\n", + " {\n", + "\n", + " \"cust_id\": cust_id,\n", + "\n", + " \"email\": emails_clean,\n", + "\n", + " \"phone\": phones_clean,\n", + "\n", + " \"signup_date\": dates_clean,\n", + "\n", + " \"amount_spent\": amounts_clean,\n", + "\n", + " \"status\": status_clean,\n", + "\n", + " \"country\": country_clean,\n", + "\n", + " }\n", + "\n", + ")\n", + "\n", + "\n", + "\n", "messy_df.head()" ] }, { "cell_type": "markdown", + "id": "8dd0d8092fe74a7c96281538738b07e2", "metadata": {}, "source": [ "## 2. Learn a profile\n\n", "\n\n", - "`fd.learn` runs the 5-stage pipeline (align \u2192 diff \u2192 classify \u2192 extract \u2192 fit/holdout-validate)\n\n", + "`fd.learn` runs the 5-stage pipeline (align → diff → classify → extract → fit/holdout-validate)\n\n", "and returns a `LearningProfile`. `context=` is free text that seeds semantic-type inference,\n\n", "same as `fd.clean(df, context=...)`. `key=\"cust_id\"` aligns messy/clean rows unambiguously." ] @@ -129,65 +207,84 @@ { "cell_type": "code", "execution_count": null, + "id": "72eea5119410473aa328ad9291626812", "metadata": {}, "outputs": [], "source": [ - "profile = fd.learn(\n\n", - " messy_df,\n\n", - " clean_df,\n\n", - " context=\"Ecommerce customer data from India\",\n\n", - " key=\"cust_id\",\n\n", - ")\n\n", - "print(profile.summary())\n\n", + "profile = fd.learn(\n", + "\n", + " messy_df,\n", + "\n", + " clean_df,\n", + "\n", + " context=\"Ecommerce customer data from India\",\n", + "\n", + " key=\"cust_id\",\n", + "\n", + ")\n", + "\n", + "print(profile.summary())\n", + "\n", "print(\"contains_raw_values:\", profile.manifest.contains_raw_values)" ] }, { "cell_type": "markdown", + "id": "8edb47106e1a46a883d545849b8ab81b", "metadata": {}, "source": [ "`contains_raw_values` is `False`: by default (`privacy=\"mask\"`), sensitive semantic types\n\n", "(`email`, `phone`, `person_name`, `national_id`, `address`, `postal_code`, `free_text`) are\n\n", "HMAC-masked before anything is stored, and no raw literal imputation values are ever learned.\n\n", "Only the non-sensitive `status` column contributes real value-map literals here (`\"Shipped\"\n\n", - "\u2192 \"shipped\"`, `\"Deliverd\" \u2192 \"delivered\"`), because `status` isn't a sensitive semantic type." + "→ \"shipped\"`, `\"Deliverd\" → \"delivered\"`), because `status` isn't a sensitive semantic type." ] }, { "cell_type": "markdown", + "id": "10185d26023b46108eb7d9f57d49d2b3", "metadata": {}, "source": [ "## 3. Save, and confirm no literal PII leaks into the file\n\n", "\n\n", "A `.fdprofile` is a zip with a manifest that hashes every member for integrity. We check the\n\n", "raw bytes of the saved file for any of the real emails or phone numbers from the training\n\n", - "data \u2014 none should appear, since those columns are masked by default." + "data — none should appear, since those columns are masked by default." ] }, { "cell_type": "code", "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", "metadata": {}, "outputs": [], "source": [ - "fd.save_profile(profile, \"/tmp/e2e_orders.fdprofile\")\n\n", - "\n\n", - "with open(\"/tmp/e2e_orders.fdprofile\", \"rb\") as f:\n\n", - " raw_bytes = f.read()\n\n", - "\n\n", - "leak_terms = emails_clean + phones_clean\n\n", - "leaks = [t for t in leak_terms if t.encode(\"utf-8\") in raw_bytes]\n\n", + "fd.save_profile(profile, \"/tmp/e2e_orders.fdprofile\")\n", + "\n", + "\n", + "\n", + "with open(\"/tmp/e2e_orders.fdprofile\", \"rb\") as f:\n", + "\n", + " raw_bytes = f.read()\n", + "\n", + "\n", + "\n", + "leak_terms = emails_clean + phones_clean\n", + "\n", + "leaks = [t for t in leak_terms if t.encode(\"utf-8\") in raw_bytes]\n", + "\n", "print(\"literal leaks found in zip bytes:\", leaks)" ] }, { "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", "metadata": {}, "source": [ "## 4. Load the profile and replay it on new data\n\n", "\n\n", "`fd.clean(df, profile=..., return_report=True)` returns a `(cleaned_df, report)` tuple.\n\n", - "Profile-backed proposals are gathered by the semantic layer, which stays off by default \u2014 pass\n\n", + "Profile-backed proposals are gathered by the semantic layer, which stays off by default — pass\n\n", "`semantic_mode=\"auto\"` (or `\"assist\"`/`\"review\"`) explicitly, exactly as you would to enable\n\n", "semantic assistance without a profile." ] @@ -195,54 +292,77 @@ { "cell_type": "code", "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", "metadata": {}, "outputs": [], "source": [ - "loaded = fd.load_profile(\"/tmp/e2e_orders.fdprofile\")\n\n", - "\n\n", - "new_df = pd.DataFrame(\n\n", - " {\n\n", - " \"cust_id\": [1001, 1002, 1003, 1004],\n\n", - " \"email\": [\n\n", - " \" NEW.USER@example.in \",\n\n", - " \"another@example.in\",\n\n", - " \"THIRD@EXAMPLE.IN\",\n\n", - " \"fourth@example.in \",\n\n", - " ],\n\n", - " \"phone\": [\"098765 43299\", \"91-9876543298\", \"+919876543297\", \"09876543296\"],\n\n", - " \"signup_date\": [\"15/03/2024\", \"2024-03-16\", \"20/03/2024\", \"2024-03-18\"],\n\n", - " \"amount_spent\": [\"\u20b91,200.00\", \"\u20b9950.50\", \"\u20b92,000.00\", \"\u20b9800.00\"],\n\n", - " \"status\": [\"Shipped\", \"Deliverd\", \"cancelled\", \"Shipped\"],\n\n", - " \"country\": [\"India\", \"N/A\", \"India\", \"India\"],\n\n", - " }\n\n", - ")\n\n", + "loaded = fd.load_profile(\"/tmp/e2e_orders.fdprofile\")\n", + "\n", + "\n", + "\n", + "new_df = pd.DataFrame(\n", + "\n", + " {\n", + "\n", + " \"cust_id\": [1001, 1002, 1003, 1004],\n", + "\n", + " \"email\": [\n", + "\n", + " \" NEW.USER@example.in \",\n", + "\n", + " \"another@example.in\",\n", + "\n", + " \"THIRD@EXAMPLE.IN\",\n", + "\n", + " \"fourth@example.in \",\n", + "\n", + " ],\n", + "\n", + " \"phone\": [\"098765 43299\", \"91-9876543298\", \"+919876543297\", \"09876543296\"],\n", + "\n", + " \"signup_date\": [\"15/03/2024\", \"2024-03-16\", \"20/03/2024\", \"2024-03-18\"],\n", + "\n", + " \"amount_spent\": [\"₹1,200.00\", \"₹950.50\", \"₹2,000.00\", \"₹800.00\"],\n", + "\n", + " \"status\": [\"Shipped\", \"Deliverd\", \"cancelled\", \"Shipped\"],\n", + "\n", + " \"country\": [\"India\", \"N/A\", \"India\", \"India\"],\n", + "\n", + " }\n", + "\n", + ")\n", + "\n", "new_df" ] }, { "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", "metadata": {}, "source": [ "### Baseline: `fd.clean(new_df)` with no profile\n\n", "\n\n", "General-purpose heuristics (phone/date/currency normalization, sentinel-to-missing) still\n\n", "apply without any profile. But `\"Shipped\"` and `\"Deliverd\"` aren't recognizable as typos by\n\n", - "generic rules \u2014 there's no dictionary that says \"Deliverd\" means \"delivered\" for this\n\n", + "generic rules — there's no dictionary that says \"Deliverd\" means \"delivered\" for this\n\n", "dataset's vocabulary. Without the learned value map, both typos pass through unfixed." ] }, { "cell_type": "code", "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", "metadata": {}, "outputs": [], "source": [ - "baseline_cleaned, _ = fd.clean(new_df, return_report=True)\n\n", + "baseline_cleaned, _ = fd.clean(new_df, return_report=True)\n", + "\n", "baseline_cleaned[[\"cust_id\", \"status\"]]" ] }, { "cell_type": "markdown", + "id": "504fb2a444614c0babb325280ed9130a", "metadata": {}, "source": [ "### With the learned profile: `fd.clean(new_df, profile=loaded, semantic_mode=\"auto\")`\n\n", @@ -254,29 +374,38 @@ { "cell_type": "code", "execution_count": null, + "id": "59bbdb311c014d738909a11f9e486628", "metadata": {}, "outputs": [], "source": [ - "cleaned, report = fd.clean(\n\n", - " new_df, profile=loaded, semantic_mode=\"auto\", return_report=True\n\n", - ")\n\n", + "cleaned, report = fd.clean(\n", + "\n", + " new_df, profile=loaded, semantic_mode=\"auto\", return_report=True\n", + "\n", + ")\n", + "\n", "cleaned[[\"cust_id\", \"status\"]]" ] }, { "cell_type": "code", "execution_count": null, + "id": "b43b363d81ae4b689946ece5c682cd59", "metadata": {}, "outputs": [], "source": [ - "actions = [a for a in report.actions if a.metadata.get(\"profile_influenced\")]\n\n", - "print(\"profile_influenced actions:\", len(actions))\n\n", - "for a in actions:\n\n", + "actions = [a for a in report.actions if a.metadata.get(\"profile_influenced\")]\n", + "\n", + "print(\"profile_influenced actions:\", len(actions))\n", + "\n", + "for a in actions:\n", + "\n", " print(a.column, a.metadata)" ] }, { "cell_type": "markdown", + "id": "8a65eabff63a45729fe45fb5ade58bdc", "metadata": {}, "source": [ "## 5. Audit the profile\n\n", @@ -284,26 +413,33 @@ "`profile.audit()` reports every rule the profile carries and what happened to any candidate\n\n", "that didn't make it into an auto-applied rule. Here, an `email_normalize` rule was proposed\n\n", "during extraction but **demoted to suggest-only** because its holdout precision (measured on\n\n", - "a held-out split of the training pairs) fell below the default `min_precision=0.98` gate \u2014\n\n", + "a held-out split of the training pairs) fell below the default `min_precision=0.98` gate —\n\n", "the fit/holdout stage is what keeps a profile from replaying a rule that doesn't generalize." ] }, { "cell_type": "code", "execution_count": null, + "id": "c3933fab20d04ec698c2621248eb3be0", "metadata": {}, "outputs": [], "source": [ - "audit = profile.audit()\n\n", - "print(f\"rule_count={audit.rule_count} rules_by_family={audit.rules_by_family}\")\n\n", - "print(f\"value_map_entries={audit.value_map_entries} masked_entries={audit.masked_entries}\")\n\n", - "print(f\"demotions={len(audit.demotions)}\")\n\n", - "for d in audit.demotions:\n\n", + "audit = profile.audit()\n", + "\n", + "print(f\"rule_count={audit.rule_count} rules_by_family={audit.rules_by_family}\")\n", + "\n", + "print(f\"value_map_entries={audit.value_map_entries} masked_entries={audit.masked_entries}\")\n", + "\n", + "print(f\"demotions={len(audit.demotions)}\")\n", + "\n", + "for d in audit.demotions:\n", + "\n", " print(\" \", d.column, d.family, d.outcome, \"-\", d.reason)" ] }, { "cell_type": "markdown", + "id": "4dd4641cc4064e0191573fe9c69df29b", "metadata": {}, "source": [ "## Recap\n\n", @@ -313,7 +449,7 @@ "- Sensitive columns are masked by default; no full rows, no raw sensitive literals, and no\n\n", " literal imputation maps are ever stored.\n\n", "- Low-support or low-precision candidates are demoted to suggest-only rather than silently\n\n", - " applied \u2014 a profile can only replay what it validated on held-out data.\n\n", + " applied — a profile can only replay what it validated on held-out data.\n\n", "- Replay is opt-in and additive: `fd.clean(df, profile=profile, semantic_mode=\"auto\")` only\n\n", " changes behavior when you ask for it; `profile=None` (the default) is byte-identical to\n\n", " today's `fd.clean`.\n\n", diff --git a/pyproject.toml b/pyproject.toml index 5f2119d7..fe9a3714 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -287,6 +287,9 @@ skip_empty = true line-length = 99 target-version = "py39" src = ["src", "tests"] +# ASV-managed benchmark sub-project: tool-generated layout and per-machine +# result artifacts; not shipped and not held to the package lint bar. +extend-exclude = ["freshdata-benchmarks"] [tool.ruff.lint] select = ["E", "F", "W", "I", "UP", "B", "SIM", "C4", "RET", "PL"] @@ -345,11 +348,14 @@ ignore = [ # readers + memory probes import lazily (PLC0415) so the base streaming path and # `import freshdata` stay dependency-free. "src/freshdata/streaming/*" = ["PLC0415"] -"benchmarks/bench_streaming.py" = ["PLC0415"] -"benchmarks/bench_outofcore.py" = ["PLC0415"] -"benchmarks/bench_native_semantic.py" = ["PLC0415"] -"benchmarks/bench_report.py" = ["PLC0415"] +"benchmarks/*" = ["PLC0415"] "benchmarks/bench_freshcore.py" = ["E402", "I001"] +# Fixture modules encode defect tables as one row per line; wrapping those +# rows destroys the tabular readability the fixtures exist for. The gold +# builder is one linear data-construction pass; splitting it adds nothing. +"benchmarks/fixtures/*" = ["E501"] +"benchmarks/fixtures/gold.py" = ["E501", "PLR0915"] +"benchmarks/harness_metrics.py" = ["PLR0915"] "tests/test_import_time.py" = ["PLC0415"] # Render layer + ReportFrame: optional viz deps and the renderer registry are # imported lazily (PLC0415) so `import freshdata` never pulls plotly/itables.