diff --git a/src/freshdata/semantic/apply.py b/src/freshdata/semantic/apply.py index 69554df0..b2e1e1a3 100644 --- a/src/freshdata/semantic/apply.py +++ b/src/freshdata/semantic/apply.py @@ -267,8 +267,28 @@ def run_semantic( from .backends import gather_proposals # noqa: PLC0415 - avoid import cycle from .consistency import run_consistency_checks # noqa: PLC0415 - avoid import cycle - ctx = build_semantic_context(df, config) - proposals = gather_proposals(df, ctx, config, memory=memory, profile=profile, report=report) + # The semantic context, experts and backends name columns by ``str(label)``. + # Map each name back to the frame's real label so non-string labels (from + # ``read_csv(header=None)`` or ``df.T``) can be indexed. Distinct labels + # that stringify alike (``1`` and ``"1"``) cannot be told apart by name, so + # they are left out of semantic repair rather than risk repairing the + # wrong column. + labels_by_name: dict[str, list[object]] = {} + for label in df.columns: + labels_by_name.setdefault(str(label), []).append(label) + ambiguous = sorted(name for name, labels in labels_by_name.items() if len(labels) > 1) + work = df + if ambiguous: + report.add_warning( + f"semantic repair skipped column(s) {ambiguous}: distinct column labels " + "share the same string name, so a repair could not be attributed to " + "one column. Rename the columns to enable semantic repair for them." + ) + work = df.loc[:, [str(label) not in ambiguous for label in df.columns]] + label_for = {str(label): label for label in work.columns} + + ctx = build_semantic_context(work, config) + proposals = gather_proposals(work, ctx, config, memory=memory, profile=profile, report=report) out = df if proposals: @@ -276,7 +296,8 @@ def run_semantic( if replacements: # Shallow copy so we never mutate the caller's frame when applying. out = df.copy(deep=False) - for col, mapping in replacements.items(): + for name, mapping in replacements.items(): + col = label_for.get(name, name) out[col] = _apply_column(out[col], mapping) # Cross-field checks route contradictions no single-column expert can see # (date ordering, unit conflicts, sensitive-column anomalies) to a human. diff --git a/src/freshdata/semantic/consistency.py b/src/freshdata/semantic/consistency.py index d38c18da..33d0cf3f 100644 --- a/src/freshdata/semantic/consistency.py +++ b/src/freshdata/semantic/consistency.py @@ -19,6 +19,7 @@ import warnings from collections.abc import Iterable, Mapping +import numpy as np import pandas as pd from ..config import CleanConfig @@ -60,10 +61,25 @@ def _rows_text(rows: Iterable[object]) -> str: return " ".join(named) +def _utc_naive(parsed: pd.Series) -> pd.Series: + """Express datetimes as naive UTC so a tz-aware column compares with a + naive one instead of raising; naive values are read as UTC.""" + if isinstance(parsed.dtype, pd.DatetimeTZDtype): + return parsed.dt.tz_convert("UTC").dt.tz_localize(None) + return parsed + + +def _utc_naive_timestamp(ts: pd.Timestamp) -> pd.Timestamp: + if ts.tzinfo is None: + return ts + return ts.tz_convert("UTC").tz_localize(None) + + def _as_datetime(series: pd.Series) -> pd.Series | None: - """Parse a column to datetimes, or ``None`` when it clearly is not one.""" + """Parse a column to naive-UTC datetimes, or ``None`` when it clearly is + not one.""" if pd.api.types.is_datetime64_any_dtype(series): - return series + return _utc_naive(series) if series.dtype != object: return None nonnull = series.dropna() @@ -73,11 +89,17 @@ def _as_datetime(series: pd.Series) -> pd.Series | None: warnings.simplefilter("ignore") try: parsed = pd.to_datetime(series, errors="coerce") + if not pd.api.types.is_datetime64_any_dtype(parsed): + # Mixed UTC offsets (or offset and naive strings) parse to an + # object column; ``utc=True`` puts every value on one clock. + parsed = pd.to_datetime(series, errors="coerce", utc=True) except (TypeError, ValueError, OverflowError): return None + if not pd.api.types.is_datetime64_any_dtype(parsed): + return None if parsed.notna().sum() / len(nonnull) < 0.6: return None - return parsed + return _utc_naive(parsed) def _modal_value(series: pd.Series) -> object: @@ -120,12 +142,18 @@ def _check_date_pair_ordering(df: pd.DataFrame, report: CleanReport) -> None: count = int(violations.sum()) if not 0 < count <= max(3, int(0.1 * n)): continue - start_modal = _modal_value(df[start_col][both]) - end_modal = _modal_value(df[end_col][both]) + # Work positionally: ``df.at[row, col]`` returns a Series when a + # row label is duplicated (e.g. after ``pd.concat``). + both_mask = both.to_numpy(dtype=bool) + start_raw = df[start_col] + end_raw = df[end_col] + start_modal = _modal_value(start_raw[both_mask]) + end_modal = _modal_value(end_raw[both_mask]) per_column: dict[str, list[object]] = {} - for row in df.index[violations]: - start_deviates = df.at[row, start_col] != start_modal - end_deviates = df.at[row, end_col] != end_modal + for pos in np.flatnonzero(violations.to_numpy(dtype=bool)): + row = df.index[pos] + start_deviates = start_raw.iloc[pos] != start_modal + end_deviates = end_raw.iloc[pos] != end_modal if end_deviates and not start_deviates: per_column.setdefault(str(end_col), []).append(row) elif start_deviates and not end_deviates: @@ -157,6 +185,8 @@ def _check_future_start_dates( reference_ts = pd.Timestamp(str(reference)) except (ValueError, TypeError): return + # Parsed columns are naive UTC (see ``_as_datetime``); compare on that clock. + reference_utc = _utc_naive_timestamp(reference_ts) for col in df.columns: if not _START_NAME.search(str(col)): continue @@ -164,7 +194,7 @@ def _check_future_start_dates( if parsed is None: continue n = int(parsed.notna().sum()) - future = parsed.notna() & (parsed > reference_ts) + future = parsed.notna() & (parsed > reference_utc) count = int(future.sum()) if n < 8 or not 0 < count <= max(3, int(0.1 * n)): continue @@ -195,9 +225,10 @@ def _check_policy_durations(df: pd.DataFrame, report: CleanReport) -> None: if str(repair_col) == str(retention_col): continue rows: list[object] = [] - for row in df.index: - retention = _duration_days(df.at[row, retention_col]) - repair = _duration_days(df.at[row, repair_col]) + pairs = zip(df.index, df[retention_col].tolist(), df[repair_col].tolist()) + for row, retention_value, repair_value in pairs: + retention = _duration_days(retention_value) + repair = _duration_days(repair_value) if retention is not None and repair is not None and repair < retention: rows.append(row) if not rows: @@ -253,20 +284,22 @@ def _check_fahrenheit_in_celsius(df: pd.DataFrame, report: CleanReport) -> None: if not _TEMP_NAME.search(str(col)): continue numeric = pd.to_numeric(df[col], errors="coerce") - nonnull = numeric.dropna() - if len(nonnull) < 8: + present = np.flatnonzero(numeric.notna().to_numpy(dtype=bool)) + if len(present) < 8: continue - flagged: list[object] = [] - for row in df.index[numeric.notna()]: - value = float(numeric.at[row]) - others = nonnull[nonnull.index != row] - if len(others) < 4 or value <= float(others.max()) + 10: - continue - converted = (value - 32.0) * 5.0 / 9.0 - if float(others.min()) - 0.5 <= converted <= float(others.max()) + 0.5: - flagged.append(row) - if not 0 < len(flagged) <= max(2, int(0.1 * len(nonnull))): + # Positional, so a duplicated row label cannot turn a cell into a + # Series. Only the column maximum can exceed every *other* value by + # more than 10 (a tied maximum never does), so it is the one candidate. + values = numeric.iloc[present].to_numpy(dtype=float) + top = int(np.argmax(values)) + value = float(values[top]) + others = np.delete(values, top) + if value <= float(others.max()) + 10: + continue + converted = (value - 32.0) * 5.0 / 9.0 + if not float(others.min()) - 0.5 <= converted <= float(others.max()) + 0.5: continue + flagged: list[object] = [df.index[present[top]]] report.add_warning( f"column '{col}': {len(flagged)} value(s) read as Fahrenheit in a " f"Celsius-ranged column (the converted value fits the column's " @@ -282,12 +315,14 @@ def _check_timezone_transitions(df: pd.DataFrame, report: CleanReport) -> None: if not _TZ_NAME.search(str(tz_col)): continue values = df[tz_col] - transitions = [ - row - for row in df.index[values.notna()] - if isinstance(values.at[row], str) - and _TZ_TRANSITION.search(values.at[row]) + # Positions, not labels: ``values.at[row]`` is a Series on a + # duplicated row label, which would silently hide the transition. + transition_positions = [ + pos + for pos, value in enumerate(values.tolist()) + if isinstance(value, str) and _TZ_TRANSITION.search(value) ] + transitions = [df.index[pos] for pos in transition_positions] n = int(values.notna().sum()) if n < 8 or not 0 < len(transitions) <= max(3, int(0.1 * n)): continue @@ -300,11 +335,12 @@ def _check_timezone_transitions(df: pd.DataFrame, report: CleanReport) -> None: for other in df.columns: if str(other) == str(tz_col): continue + other_values = df[other] window_rows = [ - row - for row in transitions - if isinstance(df.at[row, other], str) - and _WINDOW_VALUE.search(df.at[row, other]) + df.index[pos] + for pos in transition_positions + if isinstance(other_values.iloc[pos], str) + and _WINDOW_VALUE.search(other_values.iloc[pos]) ] if not window_rows: continue diff --git a/tests/test_semantic_label_and_tz_robustness.py b/tests/test_semantic_label_and_tz_robustness.py new file mode 100644 index 00000000..ef5a98df --- /dev/null +++ b/tests/test_semantic_label_and_tz_robustness.py @@ -0,0 +1,220 @@ +"""Semantic stage robustness: duplicate row labels, non-string column labels, +and tz-aware vs naive datetimes (#231, #232, #233 — semantic parts).""" + +from __future__ import annotations + +import warnings + +import pandas as pd + +import freshdata as fd +from freshdata.report import CleanReport +from freshdata.semantic.consistency import ( + _as_datetime, + _check_date_pair_ordering, + _check_fahrenheit_in_celsius, + _check_future_start_dates, + _check_policy_durations, + _check_timezone_transitions, +) + +DUP_INDEX = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + + +def _report() -> CleanReport: + return CleanReport(rows_before=10, rows_after=10, cols_before=2, cols_after=2) + + +def _clean(df: pd.DataFrame, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fd.clean(df, verbose=False, return_report=True, **kwargs) + + +# -- #231 part 1: duplicate row index labels ----------------------------------- + + +def _ordering_frame(index=None) -> pd.DataFrame: + df = pd.DataFrame( + { + "start_date": pd.date_range("2024-01-01", periods=10).astype(str), + "end_date": pd.date_range("2024-02-01", periods=10).astype(str), + }, + index=index, + ) + df.iloc[3, 1] = "2023-01-01" + return df + + +def test_clean_with_duplicate_row_labels_reports_date_ordering_break(): + _, control = _clean(_ordering_frame().reset_index(drop=True), semantic_mode="auto") + _, report = _clean(_ordering_frame(DUP_INDEX), semantic_mode="auto") + + def ordering(rep): + return [w for w in rep.warnings if "break the date ordering" in w] + + assert ordering(control) + assert len(ordering(report)) == len(ordering(control)) + # Position 3 carries the original (duplicated) label 1. + assert all("(row 1)" in w for w in ordering(report)) + + +def test_date_pair_ordering_reports_original_labels_on_duplicate_index(): + report = _report() + _check_date_pair_ordering(_ordering_frame(DUP_INDEX), report) + assert report.warnings + assert all("(row 1)" in w for w in report.warnings) + + +def test_fahrenheit_check_handles_duplicate_row_labels(): + temps = [20.0, 21.0, 22.0, 23.0, 21.0, 71.6, 22.0, 20.0, 23.0, 21.5] + unique = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}), unique) + duplicated = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}, index=DUP_INDEX), duplicated) + + assert len(unique.warnings) == 1 and "(row 5)" in unique.warnings[0] + assert len(duplicated.warnings) == 1 and "(row 2)" in duplicated.warnings[0] + assert "1 value(s)" in duplicated.warnings[0] + + +def test_fahrenheit_check_ignores_a_tied_maximum(): + temps = [20.0, 21.0, 22.0, 23.0, 21.0, 71.6, 71.6, 20.0, 23.0, 21.5] + report = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}, index=DUP_INDEX), report) + assert report.warnings == [] + + +def test_timezone_transition_found_on_duplicate_row_labels(): + df = pd.DataFrame( + { + "timezone": ["UTC"] * 9 + ["UTC→IST"], + "window": ["09:00-17:00"] * 10, + }, + index=DUP_INDEX, + ) + report = _report() + _check_timezone_transitions(df, report) + assert any("column 'timezone'" in w and "(row 4)" in w for w in report.warnings) + assert any("column 'window'" in w and "(row 4)" in w for w in report.warnings) + + +def test_policy_duration_conflict_found_on_duplicate_row_labels(): + df = pd.DataFrame( + {"retention": ["7 years", "1 year"], "purge_after": ["30 days", "2 years"]}, + index=[7, 7], + ) + report = _report() + _check_policy_durations(df, report) + assert len(report.warnings) == 2 + assert all("1 row(s) (row 7)" in w for w in report.warnings) + + +# -- #232 part 1: non-string column labels ------------------------------------- + + +def test_semantic_clean_accepts_integer_column_labels(): + df = pd.DataFrame({1: ["yes", "no", "yes", "no"], 2: ["twenty", "5", "6", "7"]}) + out, report = _clean(df, semantic_mode="auto", column_names=False) + + assert out[2].tolist() == [20, 5, 6, 7] + assert "2" not in out.columns + semantic = [a for a in report.actions if a.step == "semantic"] + assert any(a.column == "2" and a.description.startswith("Normalized") for a in semantic) + + +def test_semantic_repair_skips_labels_that_stringify_alike(): + df = pd.DataFrame({1: ["a", "b", "c", "d"], "1": ["twenty", "5", "6", "7"]}) + out, report = _clean(df, semantic_mode="auto", column_names=False) + + assert list(out.columns)[:2] == [1, "1"] + assert out["1"].tolist() == ["twenty", "5", "6", "7"] + assert any("semantic repair skipped column(s) ['1']" in w for w in report.warnings) + assert not [a for a in report.actions if a.step == "semantic" and a.column == "1"] + + +def test_semantic_clean_with_colliding_labels_still_repairs_other_columns(): + df = pd.DataFrame( + {1: ["a", "b", "c", "d"], "1": ["x", "y", "z", "w"], "qty": ["twenty", "5", "6", "7"]} + ) + out, _ = _clean(df, semantic_mode="auto", column_names=False) + assert out["qty"].tolist() == [20, 5, 6, 7] + + +# -- #233 part 1: tz-aware vs naive datetimes ---------------------------------- + + +def test_clean_with_tz_aware_start_and_naive_end_completes(): + df = pd.DataFrame( + { + "start_date": pd.date_range("2024-01-01", periods=10, tz="UTC"), + "end_date": pd.date_range("2024-02-01", periods=10), + } + ) + _clean(df, semantic_mode="auto") + + +def test_date_pair_ordering_compares_aware_and_naive_on_utc(): + start = pd.Series(pd.date_range("2024-01-01", periods=10, tz="UTC")) + end = pd.Series(pd.date_range("2024-02-01", periods=10)) + end.iloc[5] = pd.Timestamp("2023-12-01") + report = _report() + _check_date_pair_ordering(pd.DataFrame({"start_date": start, "end_date": end}), report) + assert report.warnings + assert all("(row 5)" in w for w in report.warnings) + + +def test_as_datetime_normalizes_offsets_to_naive_utc(): + aware = pd.Series(["2024-01-01T10:00:00+05:00"] * 5) + mixed = pd.Series(["2024-03-10T01:30:00-05:00", "2024-03-10T03:30:00-04:00"] * 3) + tz_column = pd.Series(pd.date_range("2024-01-01", periods=5, tz="Asia/Kolkata")) + + parsed_aware = _as_datetime(aware) + parsed_mixed = _as_datetime(mixed) + parsed_tz = _as_datetime(tz_column) + + assert parsed_aware is not None and parsed_aware.dt.tz is None + assert parsed_aware.iloc[0] == pd.Timestamp("2024-01-01 05:00:00") + assert parsed_mixed is not None and parsed_mixed.dt.tz is None + assert parsed_mixed.iloc[0] == pd.Timestamp("2024-03-10 06:30:00") + assert parsed_mixed.iloc[1] == pd.Timestamp("2024-03-10 07:30:00") + assert parsed_tz is not None and parsed_tz.dt.tz is None + assert parsed_tz.iloc[0] == pd.Timestamp("2023-12-31 18:30:00") + + +def test_date_pair_ordering_with_mixed_offset_strings(): + starts = [f"2024-03-{d:02d}T08:00:00-05:00" for d in range(1, 11)] + ends = [f"2024-03-{d:02d}T10:00:00-04:00" for d in range(1, 11)] + # 08:00-05:00 is 13:00 UTC; 12:00-04:00 would be 16:00 UTC (ordered), but + # 07:00-04:00 is 11:00 UTC, before its start. + ends[4] = "2024-03-05T07:00:00-04:00" + report = _report() + _check_date_pair_ordering(pd.DataFrame({"start_date": starts, "end_date": ends}), report) + assert report.warnings + assert all("(row 4)" in w for w in report.warnings) + + +def test_future_start_dates_with_tz_aware_column_and_naive_reference(): + starts = pd.Series(pd.date_range("2024-01-01", periods=10, tz="UTC")) + starts.iloc[7] = pd.Timestamp("2025-01-01", tz="UTC") + report = _report() + _check_future_start_dates( + pd.DataFrame({"start_date": starts}), {"reference_date": "2024-06-01"}, report + ) + assert len(report.warnings) == 1 and "(row 7)" in report.warnings[0] + assert "2024-06-01" in report.warnings[0] + + +def test_future_start_dates_with_naive_column_and_aware_reference(): + starts = pd.Series(pd.date_range("2024-05-25", periods=10)) + report = _report() + # 2024-06-01T00:00+05:30 is 2024-05-31 18:30 UTC, so only 2024-06-01 onward + # (positions 7-9) lies after it. + _check_future_start_dates( + pd.DataFrame({"start_date": starts}), + {"reference_date": "2024-06-01T00:00:00+05:30"}, + report, + ) + assert len(report.warnings) == 1 + assert "3 date(s)" in report.warnings[0] + assert "(row 7) (row 8) (row 9)" in report.warnings[0]