diff --git a/src/freshdata/learning/extract.py b/src/freshdata/learning/extract.py index 33294795..f6bdbf06 100644 --- a/src/freshdata/learning/extract.py +++ b/src/freshdata/learning/extract.py @@ -20,7 +20,7 @@ from __future__ import annotations -from collections.abc import Mapping +from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any @@ -36,6 +36,7 @@ ExamplePair, ValueMap, ValueMapEntry, + encode_value, ) __all__ = ["ExtractionResult", "extract_artifacts"] @@ -65,6 +66,37 @@ _LEARNED_TIER = 4 +def _pattern_value(value: object) -> object: + """A learned clean value as a plain JSON-native Python object. + + Aligned frames hand back numpy/pandas scalars (``int64``, ``bool_``, + ``Timestamp``, ``Decimal``...). ``json.dumps`` rejects them, so a profile + embedding them could not be saved, and ``CleaningMemory.to_json`` quietly + turned them into strings. Encode them the same way ``ValueMap`` entries are. + """ + if value is None: + return None + encoded = encode_value(value) + # encode_value keeps float/str subclasses (np.float64, np.str_) as-is. + if isinstance(encoded, bool): + return bool(encoded) + if isinstance(encoded, int): + return int(encoded) + if isinstance(encoded, float): + return float(encoded) + if isinstance(encoded, str): + return str(encoded) + return encoded + + +def _replay_patterns(entries: Iterable[ValueMapEntry]) -> dict[str, object]: + """``{raw: clean}`` replay patterns for ``CleaningMemory.value_patterns``. + + Masked entries never replay, so they are left out. + """ + return {str(e.raw_value): _pattern_value(e.clean_value) for e in entries if not e.masked} + + @dataclass class ExtractionResult: """Everything stage 4 learned, before holdout validation.""" @@ -257,7 +289,7 @@ def extract_artifacts( masked=col_masked, ) if replayable: - value_patterns[column] = {str(e.raw_value): e.clean_value for e in replayable} + value_patterns[column] = _replay_patterns(replayable) if learned_sentinels: result.config_deltas["extra_sentinels"] = tuple(sorted(set(learned_sentinels))) diff --git a/src/freshdata/learning/fit_eval.py b/src/freshdata/learning/fit_eval.py index 94b96a3a..a56b3467 100644 --- a/src/freshdata/learning/fit_eval.py +++ b/src/freshdata/learning/fit_eval.py @@ -25,7 +25,7 @@ from ..context.types import ColumnConstraint from ..semantic.formats import normalize_phone_in, repair_email from .classify import _norm_token, _parse_date # shared deterministic helpers -from .extract import ExtractionResult, extract_artifacts +from .extract import ExtractionResult, _replay_patterns, extract_artifacts from .types import ( AlignedPair, AlignmentReport, @@ -357,9 +357,7 @@ def _drop_low_precision_entries( if len(kept) != len(value_map.entries): value_map.entries = kept if final.memory is not None and column in final.memory.value_patterns: - final.memory.value_patterns[column] = { - str(e.raw_value): e.clean_value for e in kept if not e.masked - } + final.memory.value_patterns[column] = _replay_patterns(kept) if not value_map.entries: del final.value_maps[column] return records diff --git a/src/freshdata/learning/merge.py b/src/freshdata/learning/merge.py index 5f04c5ff..0501cdb9 100644 --- a/src/freshdata/learning/merge.py +++ b/src/freshdata/learning/merge.py @@ -14,6 +14,7 @@ from __future__ import annotations +import copy from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Any, Literal @@ -435,20 +436,27 @@ def _merge_examples(a: ExampleBank | None, b: ExampleBank | None) -> ExampleBank def _merge_memory(a: Any, b: Any, strategy: str) -> Any: + """Merged embedded memory: always a fresh copy, never a parent's object. + + Returning (or editing) a parent's ``CleaningMemory`` would let the merge, + or any later change to the merged profile, rewrite the parents' replay + behaviour and saved bytes. + """ if a is None: - return b + return copy.deepcopy(b) if b is None or strategy == "prefer_self": - return a + return copy.deepcopy(a) if strategy == "prefer_other": - return b + return copy.deepcopy(b) # union: keep self's memory but fold in non-conflicting value patterns. - merged_patterns = {c: dict(p) for c, p in a.value_patterns.items()} + merged = copy.deepcopy(a) + merged_patterns = {c: dict(p) for c, p in merged.value_patterns.items()} for column, patterns in b.value_patterns.items(): target = merged_patterns.setdefault(column, {}) for raw, clean in patterns.items(): if raw in target and str(target[raw]) != str(clean): del target[raw] # conflicting mapping: drop from replay elif raw not in target: - target[raw] = clean - a.value_patterns = merged_patterns - return a + target[raw] = copy.deepcopy(clean) + merged.value_patterns = merged_patterns + return merged diff --git a/src/freshdata/repairplan.py b/src/freshdata/repairplan.py index 4a671d07..52a9ca27 100644 --- a/src/freshdata/repairplan.py +++ b/src/freshdata/repairplan.py @@ -16,6 +16,7 @@ import dataclasses import hashlib import json +from collections import Counter from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from pathlib import Path @@ -587,12 +588,49 @@ def _check_drift(df: pd.DataFrame, plan: RepairPlan) -> None: ) +def _cell_counts(series: pd.Series) -> Mapping[object, int] | None: + """Occurrences of each cell value, keyed with plain ``dict`` semantics. + + ``semantic.apply._apply_column`` replaces cells via ``mapping.get(v, v)``, + so counting with the same hash/equality lookup gives exactly the number of + cells a mapping entry will replace. ``None`` when a cell is unhashable. + """ + try: + return Counter(series.tolist()) + except TypeError: + return None + + +def _count_matches( + series: pd.Series, raw: object, counts: Mapping[object, int] | None +) -> int: + """Cells of *series* that a mapping keyed on *raw* would replace.""" + if counts is not None: + return int(counts.get(raw, 0)) + probe = {raw: None} + matched = 0 + for value in series.tolist(): + try: + matched += value in probe + except TypeError: # unhashable cell: the mapping cannot match it either + continue + return matched + + def _collect_mappings( out: pd.DataFrame, plan: RepairPlan, report: CleanReport -) -> tuple[dict[str, dict[object, object]], list[PlannedAction]]: - """Group executable value repairs per column; record stale ones.""" +) -> tuple[dict[str, dict[object, object]], list[PlannedAction], dict[str, int]]: + """Group executable value repairs per column; record stale ones. + + Returns the per-column mappings, the actions that will run, and each + running action's observed match count in *out* (keyed by action id). Under + ``allow_drift=True`` an action whose raw value no longer occurs is skipped + and recorded, never reported as applied. + """ mappings: dict[str, dict[object, object]] = {} executed: list[PlannedAction] = [] + observed: dict[str, int] = {} + counts_by_column: dict[str, Mapping[object, int] | None] = {} for action in plan.actions: if not _executable(action): continue @@ -607,11 +645,26 @@ def _collect_mappings( metadata={"action_id": action.id}, ) continue - mappings.setdefault(column, {})[action.params.get("raw_value")] = ( - action.params.get("proposed_value") - ) + raw = action.params.get("raw_value") + if column not in counts_by_column: + counts_by_column[column] = _cell_counts(out[column]) + matched = _count_matches(out[column], raw, counts_by_column[column]) + if matched == 0: + report.add( + "apply_plan", + f"skipped {action.id}: value {raw!r} not present in column " + f"{column!r} (frame drift)", + column=column, + count=0, + status="skipped", + risk=action.risk, + metadata={"action_id": action.id}, + ) + continue + mappings.setdefault(column, {})[raw] = action.params.get("proposed_value") executed.append(action) - return mappings, executed + observed[action.id] = matched + return mappings, executed, observed def _capture_undo( @@ -639,8 +692,14 @@ def _capture_undo( def _record_action( - report: CleanReport, plan: RepairPlan, action: PlannedAction, *, applied: bool + report: CleanReport, + plan: RepairPlan, + action: PlannedAction, + *, + applied: bool, + count: int | None = None, ) -> None: + """Record one plan action; *count* is the observed number of cells applied.""" column = str(action.column) if action.column is not None else None metadata = { "action_id": action.id, @@ -656,7 +715,7 @@ def _record_action( f"applied {action.id}: {action.params.get('raw_value')!r} -> " f"{action.params.get('proposed_value')!r}", column=column, - count=action.n_affected, + count=action.n_affected if count is None else count, rationale=action.rationale, risk=action.risk, confidence=action.confidence, @@ -674,6 +733,10 @@ def _record_action( status = "skipped" elif action.approval == "pending": why, status = "not approved", "suggested" + elif _executable(action): + # Approved and runnable yet not run: _collect_mappings found its column + # or raw value missing (allow_drift=True) and recorded why. + why, status = "frame drift", "skipped" else: why, status = str(action.params.get("gate_reason", "skipped")), "skipped" report.add( @@ -725,12 +788,14 @@ def execute_plan( out, protected_column_set(plan.config, out.columns, include_legacy=True) ) - mappings, executed = _collect_mappings(out, plan, report) + mappings, executed, observed = _collect_mappings(out, plan, report) undo_log = _capture_undo(out, executed, undo_cell_limit) if keep_undo else None for column, mapping in mappings.items(): out[column] = _apply_column(out[column], mapping) for action in plan.actions: - _record_action(report, plan, action, applied=action in executed) + _record_action( + report, plan, action, applied=action in executed, count=observed.get(action.id) + ) verify_protected(out, guard_snapshot, report) diff --git a/tests/test_learning_and_plan_audit.py b/tests/test_learning_and_plan_audit.py new file mode 100644 index 00000000..36974830 --- /dev/null +++ b/tests/test_learning_and_plan_audit.py @@ -0,0 +1,290 @@ +"""Regression tests for learned-memory types, merge isolation and plan audit. + +* #256 — learned clean values (numpy/pandas scalars) must be JSON-native in + the embedded ``CleaningMemory`` so profiles save and round-trip. +* #257 — ``LearningProfile.merge`` must not mutate or share a parent's memory. +* #258 — ``apply_plan(allow_drift=True)`` must record actions whose raw values + are gone as skipped, and applied actions with their observed cell count. +""" + +from __future__ import annotations + +import decimal +import json +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.learning import learn, load_profile, save_profile +from freshdata.learning.extract import _pattern_value +from freshdata.learning.merge import _merge_memory +from freshdata.repairplan import _cell_counts, _count_matches, compute_decisions_hash +from freshdata.semantic.apply import _apply_column + +# --------------------------------------------------------------------------- # +# #256 — JSON-native learned replay patterns +# --------------------------------------------------------------------------- # + +LEVELS = ["low", "medium", "high"] * 10 + + +def _ordinal_pair(mapping: dict[str, object]) -> tuple[pd.DataFrame, pd.DataFrame]: + messy = pd.DataFrame({"id": range(len(LEVELS)), "priority": LEVELS}) + clean = pd.DataFrame({"id": range(len(LEVELS)), "priority": [mapping[v] for v in LEVELS]}) + return messy, clean + + +def _assert_json_native(value_patterns: dict[str, object]) -> None: + for column, patterns in value_patterns.items(): + if not isinstance(patterns, dict): + continue + for value in patterns.values(): + assert value is None or type(value) in (bool, int, float, str), ( + column, + value, + type(value), + ) + + +class TestLearnedPatternsAreJsonNative: + def test_integer_clean_values_save_load_and_replay(self, tmp_path: Path) -> None: + messy, clean = _ordinal_pair({"low": 1, "medium": 2, "high": 3}) + profile = learn(messy, clean, key="id", min_support=2) + assert profile.memory is not None + expected = {"low": 1, "medium": 2, "high": 3} + assert profile.memory.value_patterns["priority"] == expected + _assert_json_native(profile.memory.value_patterns) + # to_json used default=str, which silently turned 1 into "1". + assert json.loads(profile.memory.to_json())["value_patterns"]["priority"] == expected + + path = tmp_path / "ordinal.fdprofile" + save_profile(profile, path) # raised TypeError: int64 not JSON serializable + loaded = load_profile(path) + assert loaded.memory is not None + assert loaded.memory.to_dict() == profile.memory.to_dict() + + before = fd.clean(messy, profile=profile, semantic_mode="auto", verbose=False) + after = fd.clean(messy, profile=loaded, semantic_mode="auto", verbose=False) + pd.testing.assert_frame_equal(before, after) + + @pytest.mark.parametrize( + "mapping", + [ + {"low": 0.5, "medium": 1.5, "high": 2.5}, + {"low": False, "medium": True, "high": True}, + { + "low": pd.Timestamp("2024-01-01"), + "medium": pd.Timestamp("2024-01-02"), + "high": pd.Timestamp("2024-01-03"), + }, + { + "low": decimal.Decimal("1.1"), + "medium": decimal.Decimal("2.2"), + "high": decimal.Decimal("3.3"), + }, + ], + ids=["float", "bool", "timestamp", "decimal"], + ) + def test_other_scalar_clean_values_save( + self, tmp_path: Path, mapping: dict[str, object] + ) -> None: + messy, clean = _ordinal_pair(mapping) + profile = learn(messy, clean, key="id", min_support=2) + assert profile.memory is not None + _assert_json_native(profile.memory.value_patterns) + json.dumps(profile.memory.to_dict()) # no default= needed + + path = tmp_path / "scalars.fdprofile" + save_profile(profile, path) + loaded = load_profile(path) + assert loaded.memory is not None + assert loaded.memory.to_dict() == profile.memory.to_dict() + + @pytest.mark.parametrize( + ("value", "expected"), + [ + (np.int64(3), 3), + (np.float64(1.5), 1.5), + (np.bool_(True), True), + (np.str_("x"), "x"), + (pd.Timestamp("2024-01-02"), "2024-01-02T00:00:00"), + (decimal.Decimal("1.5"), "1.5"), + (7, 7), + ("text", "text"), + (None, None), + ], + ) + def test_pattern_value_unwraps_scalars(self, value: object, expected: object) -> None: + got = _pattern_value(value) + assert got == expected + assert type(got) is type(expected) + + +# --------------------------------------------------------------------------- # +# #257 — merge never mutates or shares a parent's CleaningMemory +# --------------------------------------------------------------------------- # + +STRATEGIES = ["union_min_precision", "prefer_self", "prefer_other", "error_on_conflict"] + + +def _status_pair(raw: str, fixed: str) -> tuple[pd.DataFrame, pd.DataFrame]: + messy = pd.DataFrame({"id": range(20), "status": [raw] * 10 + ["done"] * 10}) + clean = pd.DataFrame({"id": range(20), "status": [fixed] * 10 + ["done"] * 10}) + return messy, clean + + +@pytest.fixture() +def parents(): + p1 = learn(*_status_pair("pend-ing", "pending"), key="id", min_support=2) + p2 = learn(*_status_pair("in_prog", "in progress"), key="id", min_support=2) + assert p1.memory is not None and p2.memory is not None + return p1, p2 + + +def _memory_snapshot(profile) -> str: + return json.dumps(profile.memory.to_dict(), sort_keys=True) + + +class TestMergeLeavesParentsUntouched: + @pytest.mark.parametrize("strategy", STRATEGIES) + def test_merge_neither_mutates_nor_shares_parent_memory(self, parents, strategy) -> None: + p1, p2 = parents + before = (_memory_snapshot(p1), _memory_snapshot(p2)) + merged = p1.merge(p2, strategy=strategy) + assert (_memory_snapshot(p1), _memory_snapshot(p2)) == before + assert merged.memory is not p1.memory + assert merged.memory is not p2.memory + # Later edits to the merged profile must not leak into either parent. + merged.memory.value_patterns["status"]["zzz"] = "leaked" + merged.memory.value_patterns["new_col"] = {"a": "b"} + assert (_memory_snapshot(p1), _memory_snapshot(p2)) == before + + def test_default_union_still_folds_in_other_patterns(self, parents) -> None: + p1, p2 = parents + merged = p1.merge(p2) + assert merged.memory.value_patterns["status"] == { + "pend-ing": "pending", + "in_prog": "in progress", + } + assert p1.memory.value_patterns == {"status": {"pend-ing": "pending"}} + assert p2.memory.value_patterns == {"status": {"in_prog": "in progress"}} + + def test_saved_parent_bytes_unchanged_by_merge(self, parents, tmp_path: Path) -> None: + p1, p2 = parents + first, second = tmp_path / "before.fdprofile", tmp_path / "after.fdprofile" + save_profile(p1, first) + p1.merge(p2) + save_profile(p1, second) + assert first.read_bytes() == second.read_bytes() + + def test_merge_memory_helper_copies_every_branch(self, parents) -> None: + a, b = parents[0].memory, parents[1].memory + results = [ + _merge_memory(a, None, "union_min_precision"), + _merge_memory(None, b, "union_min_precision"), + _merge_memory(a, b, "prefer_self"), + _merge_memory(a, b, "prefer_other"), + _merge_memory(a, b, "union_min_precision"), + ] + for got in results: + assert got is not a and got is not b + for column, patterns in got.value_patterns.items(): + assert patterns is not a.value_patterns.get(column) + assert patterns is not b.value_patterns.get(column) + assert _merge_memory(None, None, "prefer_self") is None + + +# --------------------------------------------------------------------------- # +# #258 — apply_plan audit reflects what actually happened under drift +# --------------------------------------------------------------------------- # + +PLAN_RULES = "Allowed status values are active, inactive, pending." +RAW_VALUES = (" Active ", "INACTIVE", "pend-ing") + + +def _messy_status() -> pd.DataFrame: + return pd.DataFrame({"status": [" Active ", "INACTIVE", "pend-ing", "active"]}) + + +def _approved_plan(df: pd.DataFrame): + plan = fd.suggest_plan(df, context=PLAN_RULES, semantic_mode="auto", verbose=False) + repair_plan = plan.repair_plan + repair_plan.approve_all(max_risk="high") + by_raw = {a.params.get("raw_value"): a for a in repair_plan.actions} + assert set(RAW_VALUES) <= set(by_raw) + return repair_plan, by_raw + + +def _plan_entries(report) -> list: + return [a for a in report.actions if a.step == "apply_plan"] + + +class TestApplyPlanDriftAudit: + def test_all_raw_values_gone_records_skips_not_applies(self) -> None: + plan, by_raw = _approved_plan(_messy_status()) + fixed = pd.DataFrame({"status": ["active", "inactive", "pending", "active"]}) + hash_before = compute_decisions_hash(plan) + + out, report = fd.apply_plan(fixed, plan, allow_drift=True) + + pd.testing.assert_frame_equal(out, fixed) + entries = _plan_entries(report) + assert entries + assert [e for e in entries if e.status == "approved"] == [] + assert all(e.status == "skipped" and e.count == 0 for e in entries) + descriptions = [e.description for e in entries] + for raw in RAW_VALUES: + action_id = by_raw[raw].id + assert f"skipped {action_id}: value {raw!r} not present in column 'status' " \ + "(frame drift)" in descriptions + assert f"did not apply {action_id} (frame drift)" in descriptions + assert report.decisions_hash == hash_before + assert compute_decisions_hash(plan) == hash_before # plan not mutated + + def test_partial_drift_records_observed_counts_and_undo(self) -> None: + plan, by_raw = _approved_plan(_messy_status()) + active = by_raw[" Active "] + assert active.n_affected == 1 # plan-time estimate + drifted = pd.DataFrame({"status": [" Active ", "active", "pending", " Active "]}) + + out, report = fd.apply_plan(drifted, plan, allow_drift=True, keep_undo=True) + + assert out["status"].tolist() == ["active", "active", "pending", "active"] + entries = _plan_entries(report) + applied = [(e.metadata["action_id"], e.count) for e in entries if e.status == "approved"] + assert applied == [(active.id, 2)] # observed cells, not n_affected + skipped_ids = { + e.metadata["action_id"] for e in entries if e.description.startswith("skipped ") + } + assert skipped_ids == {by_raw["INACTIVE"].id, by_raw["pend-ing"].id} + assert [e["action_id"] for e in report.undo_log["entries"]] == [active.id] + + def test_without_drift_counts_match_plan_estimate(self) -> None: + df = _messy_status() + plan, by_raw = _approved_plan(df) + + out, report = fd.apply_plan(df, plan) + + assert out["status"].tolist() == ["active", "inactive", "pending", "active"] + entries = _plan_entries(report) + applied = {e.metadata["action_id"]: e.count for e in entries if e.status == "approved"} + assert applied == {by_raw[raw].id: by_raw[raw].n_affected for raw in RAW_VALUES} + assert not any("frame drift" in e.description for e in entries) + + def test_count_matches_mirrors_mapping_lookup(self) -> None: + series = pd.Series(["a", "b", "a", None, 1, 1.0, True, "1"], dtype=object) + counts = _cell_counts(series) + assert counts is not None + for raw in ("a", "b", "zzz", None, 1, "1"): + replaced = _apply_column(series, {raw: "__hit__"}) + expected = int((replaced == "__hit__").sum()) + assert _count_matches(series, raw, counts) == expected, raw + + def test_count_matches_tolerates_unhashable_cells(self) -> None: + series = pd.Series([["x"], "a", "a", {"k": 1}], dtype=object) + assert _cell_counts(series) is None + assert _count_matches(series, "a", None) == 2 + assert _count_matches(series, "zzz", None) == 0