From d8f2dc7bc97fedf207e56b1ac85a5936b5e28d8e Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:58:33 +0530 Subject: [PATCH] fix(learning): JSON-native learned patterns, copy-safe merge, drift-honest plan audit #256: fd.learn stored learned clean values in CleaningMemory.value_patterns as the numpy/pandas scalars pulled from the aligned frame (int64, bool_, Timestamp, Decimal). save_profile's canonical json.dumps rejected them, and CleaningMemory.to_json (default=str) silently turned 1 into "1". Replay patterns are now built by a shared _replay_patterns helper that encodes each clean value like ValueMap entries (encode_value) and unwraps it to plain bool/int/float/str. The holdout precision gate in fit_eval rebuilt the same patterns the same buggy way and now uses the helper too. #257: _merge_memory returned the calling profile's CleaningMemory and, for the default strategy, wrote the other profile's patterns into it, so p1.merge(p2) changed p1's replay behaviour and saved bytes. Every branch now returns a deep copy, so the merge never mutates or aliases either parent. #258: with allow_drift=True, _collect_mappings only checked that an action's column existed, and _record_action logged the plan-time n_affected, so actions whose raw values were gone were reported as applied. The actual match count is now computed with the same dict lookup _apply_column uses for replacement. Actions with no matching cells are skipped and recorded as "value ... not present (frame drift)". Applied actions record the observed count. The follow-up "did not apply" record for such actions now says "frame drift" instead of repeating the gate reason. Closes #256 Closes #257 Closes #258 --- src/freshdata/learning/extract.py | 36 +++- src/freshdata/learning/fit_eval.py | 6 +- src/freshdata/learning/merge.py | 22 +- src/freshdata/repairplan.py | 85 +++++++- tests/test_learning_and_plan_audit.py | 290 ++++++++++++++++++++++++++ 5 files changed, 416 insertions(+), 23 deletions(-) create mode 100644 tests/test_learning_and_plan_audit.py diff --git a/src/freshdata/learning/extract.py b/src/freshdata/learning/extract.py index 33294795..f6bdbf06 100644 --- a/src/freshdata/learning/extract.py +++ b/src/freshdata/learning/extract.py @@ -20,7 +20,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any @@ -36,6 +36,7 @@ ExamplePair, ValueMap, ValueMapEntry, + encode_value, ) __all__ = ["ExtractionResult", "extract_artifacts"] @@ -65,6 +66,37 @@ _LEARNED_TIER = 4 +def _pattern_value(value: object) -> object: + """A learned clean value as a plain JSON-native Python object. + + Aligned frames hand back numpy/pandas scalars (``int64``, ``bool_``, + ``Timestamp``, ``Decimal``...). ``json.dumps`` rejects them, so a profile + embedding them could not be saved, and ``CleaningMemory.to_json`` quietly + turned them into strings. Encode them the same way ``ValueMap`` entries are. + """ + if value is None: + return None + encoded = encode_value(value) + # encode_value keeps float/str subclasses (np.float64, np.str_) as-is. + if isinstance(encoded, bool): + return bool(encoded) + if isinstance(encoded, int): + return int(encoded) + if isinstance(encoded, float): + return float(encoded) + if isinstance(encoded, str): + return str(encoded) + return encoded + + +def _replay_patterns(entries: Iterable[ValueMapEntry]) -> dict[str, object]: + """``{raw: clean}`` replay patterns for ``CleaningMemory.value_patterns``. + + Masked entries never replay, so they are left out. + """ + return {str(e.raw_value): _pattern_value(e.clean_value) for e in entries if not e.masked} + + @dataclass class ExtractionResult: """Everything stage 4 learned, before holdout validation.""" @@ -257,7 +289,7 @@ def extract_artifacts( masked=col_masked, ) if replayable: - value_patterns[column] = {str(e.raw_value): e.clean_value for e in replayable} + value_patterns[column] = _replay_patterns(replayable) if learned_sentinels: result.config_deltas["extra_sentinels"] = tuple(sorted(set(learned_sentinels))) diff --git a/src/freshdata/learning/fit_eval.py b/src/freshdata/learning/fit_eval.py index 94b96a3a..a56b3467 100644 --- a/src/freshdata/learning/fit_eval.py +++ b/src/freshdata/learning/fit_eval.py @@ -25,7 +25,7 @@ from ..context.types import ColumnConstraint from ..semantic.formats import normalize_phone_in, repair_email from .classify import _norm_token, _parse_date # shared deterministic helpers -from .extract import ExtractionResult, extract_artifacts +from .extract import ExtractionResult, _replay_patterns, extract_artifacts from .types import ( AlignedPair, AlignmentReport, @@ -357,9 +357,7 @@ def _drop_low_precision_entries( if len(kept) != len(value_map.entries): value_map.entries = kept if final.memory is not None and column in final.memory.value_patterns: - final.memory.value_patterns[column] = { - str(e.raw_value): e.clean_value for e in kept if not e.masked - } + final.memory.value_patterns[column] = _replay_patterns(kept) if not value_map.entries: del final.value_maps[column] return records diff --git a/src/freshdata/learning/merge.py b/src/freshdata/learning/merge.py index 5f04c5ff..0501cdb9 100644 --- a/src/freshdata/learning/merge.py +++ b/src/freshdata/learning/merge.py @@ -14,6 +14,7 @@ from __future__ import annotations +import copy from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Literal @@ -435,20 +436,27 @@ def _merge_examples(a: ExampleBank | None, b: ExampleBank | None) -> ExampleBank def _merge_memory(a: Any, b: Any, strategy: str) -> Any: + """Merged embedded memory: always a fresh copy, never a parent's object. + + Returning (or editing) a parent's ``CleaningMemory`` would let the merge, + or any later change to the merged profile, rewrite the parents' replay + behaviour and saved bytes. + """ if a is None: - return b + return copy.deepcopy(b) if b is None or strategy == "prefer_self": - return a + return copy.deepcopy(a) if strategy == "prefer_other": - return b + return copy.deepcopy(b) # union: keep self's memory but fold in non-conflicting value patterns. - merged_patterns = {c: dict(p) for c, p in a.value_patterns.items()} + merged = copy.deepcopy(a) + merged_patterns = {c: dict(p) for c, p in merged.value_patterns.items()} for column, patterns in b.value_patterns.items(): target = merged_patterns.setdefault(column, {}) for raw, clean in patterns.items(): if raw in target and str(target[raw]) != str(clean): del target[raw] # conflicting mapping: drop from replay elif raw not in target: - target[raw] = clean - a.value_patterns = merged_patterns - return a + target[raw] = copy.deepcopy(clean) + merged.value_patterns = merged_patterns + return merged diff --git a/src/freshdata/repairplan.py b/src/freshdata/repairplan.py index 4a671d07..52a9ca27 100644 --- a/src/freshdata/repairplan.py +++ b/src/freshdata/repairplan.py @@ -16,6 +16,7 @@ import dataclasses import hashlib import json +from collections import Counter from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path @@ -587,12 +588,49 @@ def _check_drift(df: pd.DataFrame, plan: RepairPlan) -> None: ) +def _cell_counts(series: pd.Series) -> Mapping[object, int] | None: + """Occurrences of each cell value, keyed with plain ``dict`` semantics. + + ``semantic.apply._apply_column`` replaces cells via ``mapping.get(v, v)``, + so counting with the same hash/equality lookup gives exactly the number of + cells a mapping entry will replace. ``None`` when a cell is unhashable. + """ + try: + return Counter(series.tolist()) + except TypeError: + return None + + +def _count_matches( + series: pd.Series, raw: object, counts: Mapping[object, int] | None +) -> int: + """Cells of *series* that a mapping keyed on *raw* would replace.""" + if counts is not None: + return int(counts.get(raw, 0)) + probe = {raw: None} + matched = 0 + for value in series.tolist(): + try: + matched += value in probe + except TypeError: # unhashable cell: the mapping cannot match it either + continue + return matched + + def _collect_mappings( out: pd.DataFrame, plan: RepairPlan, report: CleanReport -) -> tuple[dict[str, dict[object, object]], list[PlannedAction]]: - """Group executable value repairs per column; record stale ones.""" +) -> tuple[dict[str, dict[object, object]], list[PlannedAction], dict[str, int]]: + """Group executable value repairs per column; record stale ones. + + Returns the per-column mappings, the actions that will run, and each + running action's observed match count in *out* (keyed by action id). Under + ``allow_drift=True`` an action whose raw value no longer occurs is skipped + and recorded, never reported as applied. + """ mappings: dict[str, dict[object, object]] = {} executed: list[PlannedAction] = [] + observed: dict[str, int] = {} + counts_by_column: dict[str, Mapping[object, int] | None] = {} for action in plan.actions: if not _executable(action): continue @@ -607,11 +645,26 @@ def _collect_mappings( metadata={"action_id": action.id}, ) continue - mappings.setdefault(column, {})[action.params.get("raw_value")] = ( - action.params.get("proposed_value") - ) + raw = action.params.get("raw_value") + if column not in counts_by_column: + counts_by_column[column] = _cell_counts(out[column]) + matched = _count_matches(out[column], raw, counts_by_column[column]) + if matched == 0: + report.add( + "apply_plan", + f"skipped {action.id}: value {raw!r} not present in column " + f"{column!r} (frame drift)", + column=column, + count=0, + status="skipped", + risk=action.risk, + metadata={"action_id": action.id}, + ) + continue + mappings.setdefault(column, {})[raw] = action.params.get("proposed_value") executed.append(action) - return mappings, executed + observed[action.id] = matched + return mappings, executed, observed def _capture_undo( @@ -639,8 +692,14 @@ def _capture_undo( def _record_action( - report: CleanReport, plan: RepairPlan, action: PlannedAction, *, applied: bool + report: CleanReport, + plan: RepairPlan, + action: PlannedAction, + *, + applied: bool, + count: int | None = None, ) -> None: + """Record one plan action; *count* is the observed number of cells applied.""" column = str(action.column) if action.column is not None else None metadata = { "action_id": action.id, @@ -656,7 +715,7 @@ def _record_action( f"applied {action.id}: {action.params.get('raw_value')!r} -> " f"{action.params.get('proposed_value')!r}", column=column, - count=action.n_affected, + count=action.n_affected if count is None else count, rationale=action.rationale, risk=action.risk, confidence=action.confidence, @@ -674,6 +733,10 @@ def _record_action( status = "skipped" elif action.approval == "pending": why, status = "not approved", "suggested" + elif _executable(action): + # Approved and runnable yet not run: _collect_mappings found its column + # or raw value missing (allow_drift=True) and recorded why. + why, status = "frame drift", "skipped" else: why, status = str(action.params.get("gate_reason", "skipped")), "skipped" report.add( @@ -725,12 +788,14 @@ def execute_plan( out, protected_column_set(plan.config, out.columns, include_legacy=True) ) - mappings, executed = _collect_mappings(out, plan, report) + mappings, executed, observed = _collect_mappings(out, plan, report) undo_log = _capture_undo(out, executed, undo_cell_limit) if keep_undo else None for column, mapping in mappings.items(): out[column] = _apply_column(out[column], mapping) for action in plan.actions: - _record_action(report, plan, action, applied=action in executed) + _record_action( + report, plan, action, applied=action in executed, count=observed.get(action.id) + ) verify_protected(out, guard_snapshot, report) diff --git a/tests/test_learning_and_plan_audit.py b/tests/test_learning_and_plan_audit.py new file mode 100644 index 00000000..36974830 --- /dev/null +++ b/tests/test_learning_and_plan_audit.py @@ -0,0 +1,290 @@ +"""Regression tests for learned-memory types, merge isolation and plan audit. + +* #256 — learned clean values (numpy/pandas scalars) must be JSON-native in + the embedded ``CleaningMemory`` so profiles save and round-trip. +* #257 — ``LearningProfile.merge`` must not mutate or share a parent's memory. +* #258 — ``apply_plan(allow_drift=True)`` must record actions whose raw values + are gone as skipped, and applied actions with their observed cell count. +""" + +from __future__ import annotations + +import decimal +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.learning import learn, load_profile, save_profile +from freshdata.learning.extract import _pattern_value +from freshdata.learning.merge import _merge_memory +from freshdata.repairplan import _cell_counts, _count_matches, compute_decisions_hash +from freshdata.semantic.apply import _apply_column + +# --------------------------------------------------------------------------- # +# #256 — JSON-native learned replay patterns +# --------------------------------------------------------------------------- # + +LEVELS = ["low", "medium", "high"] * 10 + + +def _ordinal_pair(mapping: dict[str, object]) -> tuple[pd.DataFrame, pd.DataFrame]: + messy = pd.DataFrame({"id": range(len(LEVELS)), "priority": LEVELS}) + clean = pd.DataFrame({"id": range(len(LEVELS)), "priority": [mapping[v] for v in LEVELS]}) + return messy, clean + + +def _assert_json_native(value_patterns: dict[str, object]) -> None: + for column, patterns in value_patterns.items(): + if not isinstance(patterns, dict): + continue + for value in patterns.values(): + assert value is None or type(value) in (bool, int, float, str), ( + column, + value, + type(value), + ) + + +class TestLearnedPatternsAreJsonNative: + def test_integer_clean_values_save_load_and_replay(self, tmp_path: Path) -> None: + messy, clean = _ordinal_pair({"low": 1, "medium": 2, "high": 3}) + profile = learn(messy, clean, key="id", min_support=2) + assert profile.memory is not None + expected = {"low": 1, "medium": 2, "high": 3} + assert profile.memory.value_patterns["priority"] == expected + _assert_json_native(profile.memory.value_patterns) + # to_json used default=str, which silently turned 1 into "1". + assert json.loads(profile.memory.to_json())["value_patterns"]["priority"] == expected + + path = tmp_path / "ordinal.fdprofile" + save_profile(profile, path) # raised TypeError: int64 not JSON serializable + loaded = load_profile(path) + assert loaded.memory is not None + assert loaded.memory.to_dict() == profile.memory.to_dict() + + before = fd.clean(messy, profile=profile, semantic_mode="auto", verbose=False) + after = fd.clean(messy, profile=loaded, semantic_mode="auto", verbose=False) + pd.testing.assert_frame_equal(before, after) + + @pytest.mark.parametrize( + "mapping", + [ + {"low": 0.5, "medium": 1.5, "high": 2.5}, + {"low": False, "medium": True, "high": True}, + { + "low": pd.Timestamp("2024-01-01"), + "medium": pd.Timestamp("2024-01-02"), + "high": pd.Timestamp("2024-01-03"), + }, + { + "low": decimal.Decimal("1.1"), + "medium": decimal.Decimal("2.2"), + "high": decimal.Decimal("3.3"), + }, + ], + ids=["float", "bool", "timestamp", "decimal"], + ) + def test_other_scalar_clean_values_save( + self, tmp_path: Path, mapping: dict[str, object] + ) -> None: + messy, clean = _ordinal_pair(mapping) + profile = learn(messy, clean, key="id", min_support=2) + assert profile.memory is not None + _assert_json_native(profile.memory.value_patterns) + json.dumps(profile.memory.to_dict()) # no default= needed + + path = tmp_path / "scalars.fdprofile" + save_profile(profile, path) + loaded = load_profile(path) + assert loaded.memory is not None + assert loaded.memory.to_dict() == profile.memory.to_dict() + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (np.int64(3), 3), + (np.float64(1.5), 1.5), + (np.bool_(True), True), + (np.str_("x"), "x"), + (pd.Timestamp("2024-01-02"), "2024-01-02T00:00:00"), + (decimal.Decimal("1.5"), "1.5"), + (7, 7), + ("text", "text"), + (None, None), + ], + ) + def test_pattern_value_unwraps_scalars(self, value: object, expected: object) -> None: + got = _pattern_value(value) + assert got == expected + assert type(got) is type(expected) + + +# --------------------------------------------------------------------------- # +# #257 — merge never mutates or shares a parent's CleaningMemory +# --------------------------------------------------------------------------- # + +STRATEGIES = ["union_min_precision", "prefer_self", "prefer_other", "error_on_conflict"] + + +def _status_pair(raw: str, fixed: str) -> tuple[pd.DataFrame, pd.DataFrame]: + messy = pd.DataFrame({"id": range(20), "status": [raw] * 10 + ["done"] * 10}) + clean = pd.DataFrame({"id": range(20), "status": [fixed] * 10 + ["done"] * 10}) + return messy, clean + + +@pytest.fixture() +def parents(): + p1 = learn(*_status_pair("pend-ing", "pending"), key="id", min_support=2) + p2 = learn(*_status_pair("in_prog", "in progress"), key="id", min_support=2) + assert p1.memory is not None and p2.memory is not None + return p1, p2 + + +def _memory_snapshot(profile) -> str: + return json.dumps(profile.memory.to_dict(), sort_keys=True) + + +class TestMergeLeavesParentsUntouched: + @pytest.mark.parametrize("strategy", STRATEGIES) + def test_merge_neither_mutates_nor_shares_parent_memory(self, parents, strategy) -> None: + p1, p2 = parents + before = (_memory_snapshot(p1), _memory_snapshot(p2)) + merged = p1.merge(p2, strategy=strategy) + assert (_memory_snapshot(p1), _memory_snapshot(p2)) == before + assert merged.memory is not p1.memory + assert merged.memory is not p2.memory + # Later edits to the merged profile must not leak into either parent. + merged.memory.value_patterns["status"]["zzz"] = "leaked" + merged.memory.value_patterns["new_col"] = {"a": "b"} + assert (_memory_snapshot(p1), _memory_snapshot(p2)) == before + + def test_default_union_still_folds_in_other_patterns(self, parents) -> None: + p1, p2 = parents + merged = p1.merge(p2) + assert merged.memory.value_patterns["status"] == { + "pend-ing": "pending", + "in_prog": "in progress", + } + assert p1.memory.value_patterns == {"status": {"pend-ing": "pending"}} + assert p2.memory.value_patterns == {"status": {"in_prog": "in progress"}} + + def test_saved_parent_bytes_unchanged_by_merge(self, parents, tmp_path: Path) -> None: + p1, p2 = parents + first, second = tmp_path / "before.fdprofile", tmp_path / "after.fdprofile" + save_profile(p1, first) + p1.merge(p2) + save_profile(p1, second) + assert first.read_bytes() == second.read_bytes() + + def test_merge_memory_helper_copies_every_branch(self, parents) -> None: + a, b = parents[0].memory, parents[1].memory + results = [ + _merge_memory(a, None, "union_min_precision"), + _merge_memory(None, b, "union_min_precision"), + _merge_memory(a, b, "prefer_self"), + _merge_memory(a, b, "prefer_other"), + _merge_memory(a, b, "union_min_precision"), + ] + for got in results: + assert got is not a and got is not b + for column, patterns in got.value_patterns.items(): + assert patterns is not a.value_patterns.get(column) + assert patterns is not b.value_patterns.get(column) + assert _merge_memory(None, None, "prefer_self") is None + + +# --------------------------------------------------------------------------- # +# #258 — apply_plan audit reflects what actually happened under drift +# --------------------------------------------------------------------------- # + +PLAN_RULES = "Allowed status values are active, inactive, pending." +RAW_VALUES = (" Active ", "INACTIVE", "pend-ing") + + +def _messy_status() -> pd.DataFrame: + return pd.DataFrame({"status": [" Active ", "INACTIVE", "pend-ing", "active"]}) + + +def _approved_plan(df: pd.DataFrame): + plan = fd.suggest_plan(df, context=PLAN_RULES, semantic_mode="auto", verbose=False) + repair_plan = plan.repair_plan + repair_plan.approve_all(max_risk="high") + by_raw = {a.params.get("raw_value"): a for a in repair_plan.actions} + assert set(RAW_VALUES) <= set(by_raw) + return repair_plan, by_raw + + +def _plan_entries(report) -> list: + return [a for a in report.actions if a.step == "apply_plan"] + + +class TestApplyPlanDriftAudit: + def test_all_raw_values_gone_records_skips_not_applies(self) -> None: + plan, by_raw = _approved_plan(_messy_status()) + fixed = pd.DataFrame({"status": ["active", "inactive", "pending", "active"]}) + hash_before = compute_decisions_hash(plan) + + out, report = fd.apply_plan(fixed, plan, allow_drift=True) + + pd.testing.assert_frame_equal(out, fixed) + entries = _plan_entries(report) + assert entries + assert [e for e in entries if e.status == "approved"] == [] + assert all(e.status == "skipped" and e.count == 0 for e in entries) + descriptions = [e.description for e in entries] + for raw in RAW_VALUES: + action_id = by_raw[raw].id + assert f"skipped {action_id}: value {raw!r} not present in column 'status' " \ + "(frame drift)" in descriptions + assert f"did not apply {action_id} (frame drift)" in descriptions + assert report.decisions_hash == hash_before + assert compute_decisions_hash(plan) == hash_before # plan not mutated + + def test_partial_drift_records_observed_counts_and_undo(self) -> None: + plan, by_raw = _approved_plan(_messy_status()) + active = by_raw[" Active "] + assert active.n_affected == 1 # plan-time estimate + drifted = pd.DataFrame({"status": [" Active ", "active", "pending", " Active "]}) + + out, report = fd.apply_plan(drifted, plan, allow_drift=True, keep_undo=True) + + assert out["status"].tolist() == ["active", "active", "pending", "active"] + entries = _plan_entries(report) + applied = [(e.metadata["action_id"], e.count) for e in entries if e.status == "approved"] + assert applied == [(active.id, 2)] # observed cells, not n_affected + skipped_ids = { + e.metadata["action_id"] for e in entries if e.description.startswith("skipped ") + } + assert skipped_ids == {by_raw["INACTIVE"].id, by_raw["pend-ing"].id} + assert [e["action_id"] for e in report.undo_log["entries"]] == [active.id] + + def test_without_drift_counts_match_plan_estimate(self) -> None: + df = _messy_status() + plan, by_raw = _approved_plan(df) + + out, report = fd.apply_plan(df, plan) + + assert out["status"].tolist() == ["active", "inactive", "pending", "active"] + entries = _plan_entries(report) + applied = {e.metadata["action_id"]: e.count for e in entries if e.status == "approved"} + assert applied == {by_raw[raw].id: by_raw[raw].n_affected for raw in RAW_VALUES} + assert not any("frame drift" in e.description for e in entries) + + def test_count_matches_mirrors_mapping_lookup(self) -> None: + series = pd.Series(["a", "b", "a", None, 1, 1.0, True, "1"], dtype=object) + counts = _cell_counts(series) + assert counts is not None + for raw in ("a", "b", "zzz", None, 1, "1"): + replaced = _apply_column(series, {raw: "__hit__"}) + expected = int((replaced == "__hit__").sum()) + assert _count_matches(series, raw, counts) == expected, raw + + def test_count_matches_tolerates_unhashable_cells(self) -> None: + series = pd.Series([["x"], "a", "a", {"k": 1}], dtype=object) + assert _cell_counts(series) is None + assert _count_matches(series, "a", None) == 2 + assert _count_matches(series, "zzz", None) == 0