Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 34 additions & 2 deletions src/freshdata/learning/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

from __future__ import annotations

from collections.abc import Mapping
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any

Expand All @@ -36,6 +36,7 @@
ExamplePair,
ValueMap,
ValueMapEntry,
encode_value,
)

__all__ = ["ExtractionResult", "extract_artifacts"]
Expand Down Expand Up @@ -65,6 +66,37 @@
_LEARNED_TIER = 4


def _pattern_value(value: object) -> object:
"""A learned clean value as a plain JSON-native Python object.

Aligned frames hand back numpy/pandas scalars (``int64``, ``bool_``,
``Timestamp``, ``Decimal``...). ``json.dumps`` rejects them, so a profile
embedding them could not be saved, and ``CleaningMemory.to_json`` quietly
turned them into strings. Encode them the same way ``ValueMap`` entries are.
"""
if value is None:
return None
encoded = encode_value(value)
# encode_value keeps float/str subclasses (np.float64, np.str_) as-is.
if isinstance(encoded, bool):
return bool(encoded)
if isinstance(encoded, int):
return int(encoded)
if isinstance(encoded, float):
return float(encoded)
if isinstance(encoded, str):
return str(encoded)
return encoded


def _replay_patterns(entries: Iterable[ValueMapEntry]) -> dict[str, object]:
"""``{raw: clean}`` replay patterns for ``CleaningMemory.value_patterns``.

Masked entries never replay, so they are left out.
"""
return {str(e.raw_value): _pattern_value(e.clean_value) for e in entries if not e.masked}


@dataclass
class ExtractionResult:
"""Everything stage 4 learned, before holdout validation."""
Expand Down Expand Up @@ -257,7 +289,7 @@ def extract_artifacts(
masked=col_masked,
)
if replayable:
value_patterns[column] = {str(e.raw_value): e.clean_value for e in replayable}
value_patterns[column] = _replay_patterns(replayable)

if learned_sentinels:
result.config_deltas["extra_sentinels"] = tuple(sorted(set(learned_sentinels)))
Expand Down
6 changes: 2 additions & 4 deletions src/freshdata/learning/fit_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from ..context.types import ColumnConstraint
from ..semantic.formats import normalize_phone_in, repair_email
from .classify import _norm_token, _parse_date # shared deterministic helpers
from .extract import ExtractionResult, extract_artifacts
from .extract import ExtractionResult, _replay_patterns, extract_artifacts
from .types import (
AlignedPair,
AlignmentReport,
Expand Down Expand Up @@ -357,9 +357,7 @@ def _drop_low_precision_entries(
if len(kept) != len(value_map.entries):
value_map.entries = kept
if final.memory is not None and column in final.memory.value_patterns:
final.memory.value_patterns[column] = {
str(e.raw_value): e.clean_value for e in kept if not e.masked
}
final.memory.value_patterns[column] = _replay_patterns(kept)
if not value_map.entries:
del final.value_maps[column]
return records
Expand Down
22 changes: 15 additions & 7 deletions src/freshdata/learning/merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

from __future__ import annotations

import copy
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Literal
Expand Down Expand Up @@ -435,20 +436,27 @@ def _merge_examples(a: ExampleBank | None, b: ExampleBank | None) -> ExampleBank


def _merge_memory(a: Any, b: Any, strategy: str) -> Any:
"""Merged embedded memory: always a fresh copy, never a parent's object.

Returning (or editing) a parent's ``CleaningMemory`` would let the merge,
or any later change to the merged profile, rewrite the parents' replay
behaviour and saved bytes.
"""
if a is None:
return b
return copy.deepcopy(b)
if b is None or strategy == "prefer_self":
return a
return copy.deepcopy(a)
if strategy == "prefer_other":
return b
return copy.deepcopy(b)
# union: keep self's memory but fold in non-conflicting value patterns.
merged_patterns = {c: dict(p) for c, p in a.value_patterns.items()}
merged = copy.deepcopy(a)
merged_patterns = {c: dict(p) for c, p in merged.value_patterns.items()}
for column, patterns in b.value_patterns.items():
target = merged_patterns.setdefault(column, {})
for raw, clean in patterns.items():
if raw in target and str(target[raw]) != str(clean):
del target[raw] # conflicting mapping: drop from replay
elif raw not in target:
target[raw] = clean
a.value_patterns = merged_patterns
return a
target[raw] = copy.deepcopy(clean)
merged.value_patterns = merged_patterns
return merged
85 changes: 75 additions & 10 deletions src/freshdata/repairplan.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import dataclasses
import hashlib
import json
from collections import Counter
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from pathlib import Path
Expand Down Expand Up @@ -587,12 +588,49 @@ def _check_drift(df: pd.DataFrame, plan: RepairPlan) -> None:
)


def _cell_counts(series: pd.Series) -> Mapping[object, int] | None:
"""Occurrences of each cell value, keyed with plain ``dict`` semantics.

``semantic.apply._apply_column`` replaces cells via ``mapping.get(v, v)``,
so counting with the same hash/equality lookup gives exactly the number of
cells a mapping entry will replace. ``None`` when a cell is unhashable.
"""
try:
return Counter(series.tolist())
except TypeError:
return None


def _count_matches(
series: pd.Series, raw: object, counts: Mapping[object, int] | None
) -> int:
"""Cells of *series* that a mapping keyed on *raw* would replace."""
if counts is not None:
return int(counts.get(raw, 0))
probe = {raw: None}
matched = 0
for value in series.tolist():
try:
matched += value in probe
except TypeError: # unhashable cell: the mapping cannot match it either
continue
return matched


def _collect_mappings(
out: pd.DataFrame, plan: RepairPlan, report: CleanReport
) -> tuple[dict[str, dict[object, object]], list[PlannedAction]]:
"""Group executable value repairs per column; record stale ones."""
) -> tuple[dict[str, dict[object, object]], list[PlannedAction], dict[str, int]]:
"""Group executable value repairs per column; record stale ones.

Returns the per-column mappings, the actions that will run, and each
running action's observed match count in *out* (keyed by action id). Under
``allow_drift=True`` an action whose raw value no longer occurs is skipped
and recorded, never reported as applied.
"""
mappings: dict[str, dict[object, object]] = {}
executed: list[PlannedAction] = []
observed: dict[str, int] = {}
counts_by_column: dict[str, Mapping[object, int] | None] = {}
for action in plan.actions:
if not _executable(action):
continue
Expand All @@ -607,11 +645,26 @@ def _collect_mappings(
metadata={"action_id": action.id},
)
continue
mappings.setdefault(column, {})[action.params.get("raw_value")] = (
action.params.get("proposed_value")
)
raw = action.params.get("raw_value")
if column not in counts_by_column:
counts_by_column[column] = _cell_counts(out[column])
matched = _count_matches(out[column], raw, counts_by_column[column])
if matched == 0:
report.add(
"apply_plan",
f"skipped {action.id}: value {raw!r} not present in column "
f"{column!r} (frame drift)",
column=column,
count=0,
status="skipped",
risk=action.risk,
metadata={"action_id": action.id},
)
continue
mappings.setdefault(column, {})[raw] = action.params.get("proposed_value")
executed.append(action)
return mappings, executed
observed[action.id] = matched
return mappings, executed, observed


def _capture_undo(
Expand Down Expand Up @@ -639,8 +692,14 @@ def _capture_undo(


def _record_action(
report: CleanReport, plan: RepairPlan, action: PlannedAction, *, applied: bool
report: CleanReport,
plan: RepairPlan,
action: PlannedAction,
*,
applied: bool,
count: int | None = None,
) -> None:
"""Record one plan action; *count* is the observed number of cells applied."""
column = str(action.column) if action.column is not None else None
metadata = {
"action_id": action.id,
Expand All @@ -656,7 +715,7 @@ def _record_action(
f"applied {action.id}: {action.params.get('raw_value')!r} -> "
f"{action.params.get('proposed_value')!r}",
column=column,
count=action.n_affected,
count=action.n_affected if count is None else count,
rationale=action.rationale,
risk=action.risk,
confidence=action.confidence,
Expand All @@ -674,6 +733,10 @@ def _record_action(
status = "skipped"
elif action.approval == "pending":
why, status = "not approved", "suggested"
elif _executable(action):
# Approved and runnable yet not run: _collect_mappings found its column
# or raw value missing (allow_drift=True) and recorded why.
why, status = "frame drift", "skipped"
else:
why, status = str(action.params.get("gate_reason", "skipped")), "skipped"
report.add(
Expand Down Expand Up @@ -725,12 +788,14 @@ def execute_plan(
out, protected_column_set(plan.config, out.columns, include_legacy=True)
)

mappings, executed = _collect_mappings(out, plan, report)
mappings, executed, observed = _collect_mappings(out, plan, report)
undo_log = _capture_undo(out, executed, undo_cell_limit) if keep_undo else None
for column, mapping in mappings.items():
out[column] = _apply_column(out[column], mapping)
for action in plan.actions:
_record_action(report, plan, action, applied=action in executed)
_record_action(
report, plan, action, applied=action in executed, count=observed.get(action.id)
)

verify_protected(out, guard_snapshot, report)

Expand Down
Loading
Loading