Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
# mypy targets Python 3.10; NumPy 2.5+ stubs require Python 3.12.
pip install -e ".[dev,ml]" "numpy<2.5"
- name: Lint
run: ruff check src tests
run: ruff check .
- name: Typecheck
run: mypy src/freshdata
- name: Test (required fast lane)
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ jobs:
# mypy targets Python 3.10; NumPy 2.5+ stubs require Python 3.12.
pip install -e ".[dev,ml,docs]" "numpy<2.5"
- name: Lint
run: ruff check src tests
run: ruff check .
- name: Typecheck
run: mypy src/freshdata
- name: Test (release fast lane)
Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,13 @@ adheres to [Semantic Versioning](https://semver.org/).
(analyze → mask → clean under a compiled policy → merge category variants →
re-score trust), and a new docs guide (`docs/ai-copilot.md`).

### Changed
- Lint now covers the whole repository (`ruff check .` in CI, closing #54):
benchmark and notebook lint debt fixed, dead code removed
(`harness_metrics` unused gold-labels block), and the ASV-managed
`freshdata-benchmarks/` sub-project excluded as tool-generated. No
runtime behavior changes.

## [1.1.1] - 2026-07-06

### Fixed
Expand Down
4 changes: 1 addition & 3 deletions benchmarks/baselines/pandas_baseline.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@

import re

import numpy as np
import pandas as pd

from . import count_authored_lines
Expand Down Expand Up @@ -50,8 +49,7 @@ def run(df: pd.DataFrame) -> pd.DataFrame:
for col in out.select_dtypes(include="number").columns:
if out[col].isna().any():
out[col] = out[col].fillna(out[col].median())
out = out.drop_duplicates().reset_index(drop=True)
return out
return out.drop_duplicates().reset_index(drop=True)


AUTHORED_LINES: int = count_authored_lines(run)
3 changes: 1 addition & 2 deletions benchmarks/baselines/pyjanitor_baseline.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,7 @@ def run(df: pd.DataFrame) -> pd.DataFrame:
out[col] = coerced
for col in out.select_dtypes(include="number").columns:
out[col] = out[col].fillna(out[col].median())
out = out.drop_duplicates().reset_index(drop=True)
return out
return out.drop_duplicates().reset_index(drop=True)


AUTHORED_LINES: int = count_authored_lines(run)
37 changes: 25 additions & 12 deletions benchmarks/bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@
if str(HERE) not in sys.path:
sys.path.insert(0, str(HERE))

import freshdata as fd # noqa: E402

import harness_metrics as hm # noqa: E402
from results_schema import SCHEMA_VERSION # noqa: E402

import freshdata as fd # noqa: E402
from fixtures import REGISTRY # noqa: E402
from results_schema import RESULTS_SCHEMA, SCHEMA_VERSION # noqa: E402

RESULTS_DIR = HERE / "results"

Expand Down Expand Up @@ -143,13 +143,16 @@ def _write_result(run_id: str, result: dict) -> Path:
def cmd_run(args: argparse.Namespace) -> int:
run_id = _now()
fixtures = args.fixtures or list(DEFAULT_SIZES)
print(f"benchmark run {run_id} freshdata={fd.__version__} mode={'aggressive' if args.aggressive else 'balanced'}")
mode = "aggressive" if args.aggressive else "balanced"
print(f"benchmark run {run_id} freshdata={fd.__version__} mode={mode}")
summary_rows = []
for name in fixtures:
size = args.size or DEFAULT_SIZES[name]
result = run_single(name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat)
result = run_single(
name, size, seed=args.seed, aggressive=args.aggressive, repeat=args.repeat
)
result["run_id"] = run_id
path = _write_result(run_id, result)
_write_result(run_id, result)
m = result["metrics"]
print(f" {name:12s} n={result['n_rows']:>8,} cols={result['n_cols']:>4} "
f"p50={m['wall_clock_p50_sec']:.3f}s peak={m['peak_memory_mb']:.1f}MB "
Expand Down Expand Up @@ -182,7 +185,10 @@ def cmd_compare(args: argparse.Namespace) -> int:
config = hm.config_for(name, df)

print(f"compare on {name} n={len(df):,} cols={df.shape[1]}\n")
header = f"{'tool':24s} {'n_rows':>8} {'n_cols':>6} {'p50_sec':>8} {'p95_sec':>8} {'peak_mb':>8} mode"
header = (
f"{'tool':24s} {'n_rows':>8} {'n_cols':>6} "
f"{'p50_sec':>8} {'p95_sec':>8} {'peak_mb':>8} mode"
)
print(header)
print("-" * len(header))

Expand Down Expand Up @@ -258,16 +264,20 @@ def _render_markdown(run_id: str, results: list[dict]) -> str:
f"- python: `{results[0]['python_version'] if results else '?'}`",
f"- platform: `{results[0]['platform'] if results else '?'}`",
"",
"| fixture | n_rows | n_cols | p50 s | p95 s | peak MB | repair % | false-repair % | preserve % | trust | monotonic | export % |",
"| fixture | n_rows | n_cols | p50 s | p95 s | peak MB | repair % "
"| false-repair % | preserve % | trust | monotonic | export % |",
"|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|:--:|--:|",
]
for r in sorted(results, key=lambda x: x["fixture"]):
m = r["metrics"]
lines.append(
f"| {r['fixture']} | {r['n_rows']:,} | {r['n_cols']} | "
f"{m['wall_clock_p50_sec']:.3f} | {m['wall_clock_p95_sec']:.3f} | {m['peak_memory_mb']:.1f} | "
f"{m['repair_fidelity_pct']} | {m['false_repair_rate_pct']} | {m['preservation_rate_pct']} | "
f"{m['trust_score']} | {'✅' if m['trust_monotonic_valid'] else '❌'} | {m['export_completeness_pct']} |"
f"{m['wall_clock_p50_sec']:.3f} | {m['wall_clock_p95_sec']:.3f} | "
f"{m['peak_memory_mb']:.1f} | "
f"{m['repair_fidelity_pct']} | {m['false_repair_rate_pct']} | "
f"{m['preservation_rate_pct']} | "
f"{m['trust_score']} | {'✅' if m['trust_monotonic_valid'] else '❌'} | "
f"{m['export_completeness_pct']} |"
)
lines += ["", "## Authored-code reduction (Metric 6)", ""]
if results:
Expand All @@ -291,7 +301,10 @@ def cmd_fixtures(args: argparse.Namespace) -> int:
bundle = mod.generate(size, seed=args.seed)
bundle.dirty_df.to_csv(out / "gold_dirty.csv", index=False)
bundle.clean_df.to_csv(out / "gold_clean.csv", index=False)
print(f" gold -> gold_dirty.csv ({bundle.dirty_df.shape}), gold_clean.csv ({bundle.clean_df.shape})")
print(
f" gold -> gold_dirty.csv ({bundle.dirty_df.shape}), "
f"gold_clean.csv ({bundle.clean_df.shape})"
)
else:
df = mod.generate(size, seed=args.seed)
path = out / f"{name}.csv"
Expand Down
11 changes: 10 additions & 1 deletion benchmarks/fixtures/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,4 +29,13 @@
#: Fixtures whose ``generate`` returns a plain DataFrame (everything but gold).
FRAME_FIXTURES = ("crm", "finance", "event_log", "wide_schema", "provenance")

__all__ = ["REGISTRY", "FRAME_FIXTURES", *REGISTRY.keys()]
__all__ = [
"REGISTRY",
"FRAME_FIXTURES",
"crm",
"finance",
"event_log",
"wide_schema",
"provenance",
"gold",
]
7 changes: 3 additions & 4 deletions benchmarks/fixtures/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,11 @@

from __future__ import annotations

from dataclasses import asdict, dataclass, field
from dataclasses import asdict, dataclass
from datetime import date
from typing import Any

import numpy as np
import pandas as pd

# -- roles -----------------------------------------------------------------
# These mirror the roles FreshData's decision engine infers
Expand Down Expand Up @@ -134,8 +133,8 @@ def uuid_series(rng: np.random.Generator, n: int, prefix: str = "") -> list[str]
hi = rng.integers(0, 2**32, size=n, dtype=np.uint64)
lo = rng.integers(0, 2**32, size=n, dtype=np.uint64)
out = []
for h, l in zip(hi.tolist(), lo.tolist()):
out.append(f"{prefix}{h:08x}-{l:08x}")
for high, low in zip(hi.tolist(), lo.tolist()):
out.append(f"{prefix}{high:08x}-{low:08x}")
return out


Expand Down
13 changes: 5 additions & 8 deletions benchmarks/fixtures/crm.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,13 @@
ACCOUNT_STATUS_REF,
BAD_COUNTRY,
COUNTRY_REF,
Defect,
GoldLabel,
ROLE_BOOL,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TEXT,
SENTINELS,
Defect,
GoldLabel,
defect_mask,
format_iso_date,
gold_to_records,
Expand Down Expand Up @@ -69,8 +67,8 @@
def _names(rng: np.random.Generator, n: int) -> np.ndarray:
f = pick(rng, _FIRST, n)
m = pick(rng, _MIDDLE, n)
l = pick(rng, _LAST, n)
return np.array([f"{a} {b} {c}" for a, b, c in zip(f, m, l)], dtype=object)
last = pick(rng, _LAST, n)
return np.array([f"{a} {b} {c}" for a, b, c in zip(f, m, last)], dtype=object)


def generate(n_rows: int, seed: int = 42, defect_rate: float | None = None) -> pd.DataFrame:
Expand Down Expand Up @@ -109,8 +107,7 @@ def generate(n_rows: int, seed: int = 42, defect_rate: float | None = None) -> p
data[col] = np.array([str(x) for x in d], dtype=object)

df = pd.DataFrame(data)
df = _inject(df, rng, defect_rate)
return df
return _inject(df, rng, defect_rate)


def _inject(df: pd.DataFrame, rng: np.random.Generator, defect_rate: float | None) -> pd.DataFrame:
Expand Down
5 changes: 2 additions & 3 deletions benchmarks/fixtures/event_log.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@
from ._common import (
BAD_OPERATION,
OPERATION_REF,
Defect,
GoldLabel,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TEXT,
Defect,
GoldLabel,
defect_mask,
gold_to_records,
manifest_to_records,
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/fixtures/finance.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@
BAD_CURRENCY,
CURRENCY_REF,
DEBIT_CREDIT_REF,
Defect,
GoldLabel,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TEXT,
Defect,
GoldLabel,
defect_mask,
format_iso_date,
gold_to_records,
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/fixtures/gold.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@
import pandas as pd

from ._common import (
Defect,
GoldLabel,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TARGET,
ROLE_TEXT,
Defect,
GoldLabel,
defect_mask,
format_iso_date,
gold_to_records,
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/fixtures/provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@
import pandas as pd

from ._common import (
Defect,
GoldLabel,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TEXT,
Defect,
GoldLabel,
defect_mask,
gold_to_records,
manifest_to_records,
Expand Down
4 changes: 2 additions & 2 deletions benchmarks/fixtures/wide_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,15 @@
import pandas as pd

from ._common import (
Defect,
GoldLabel,
ROLE_BOOL,
ROLE_CATEGORICAL,
ROLE_DATETIME,
ROLE_ID,
ROLE_NUMERIC,
ROLE_TARGET,
ROLE_TEXT,
Defect,
GoldLabel,
defect_mask,
format_iso_date,
gold_to_records,
Expand Down
Loading
Loading