From 6e2b10385d41d63ef3b7f49f3a3904bca979e710 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:52:25 +0530 Subject: [PATCH] fix(semantic): handle duplicate row labels, non-string column labels and tz-mixed dates Duplicate row index labels (#231): the cross-field checks read cells with df.at[row, col] / series.at[row], which return a Series when a row label is duplicated (common after pd.concat). The date-ordering check then raised "truth value of a Series is ambiguous" and aborted fd.clean. The Fahrenheit, timezone-transition and policy-duration checks share the same pattern; they silently missed findings instead of crashing. All four now iterate positionally and still report the original row labels. The Fahrenheit check tests only the column maximum, which is the only value that can exceed every other value by more than 10. The result is the same as the old per-row scan, in O(n). Non-string column labels (#232): the semantic context and every backend name columns str(label), and run_semantic indexed the frame with that string, so integer labels raised KeyError. run_semantic now maps each name back to the real label. Distinct labels that stringify alike (1 and "1") cannot be attributed by name, so those columns are left out of semantic repair and a report warning names them. tz-aware vs naive datetimes (#233): the date-ordering and future-start checks compared a tz-aware column with a naive column or reference timestamp, which raised TypeError. Parsed columns and the reference date are now normalized to naive UTC, with naive values read as UTC. When strings with mixed UTC offsets parse to an object column, parsing is retried with utc=True. The reported reference date is still the declared date. Refs #231 (part 1: semantic cross-field checks) Refs #232 (part 1: run_semantic column labels) Refs #233 (part 1: semantic date comparisons) --- src/freshdata/semantic/apply.py | 27 ++- src/freshdata/semantic/consistency.py | 102 +++++--- .../test_semantic_label_and_tz_robustness.py | 220 ++++++++++++++++++ 3 files changed, 313 insertions(+), 36 deletions(-) create mode 100644 tests/test_semantic_label_and_tz_robustness.py diff --git a/src/freshdata/semantic/apply.py b/src/freshdata/semantic/apply.py index 69554df0..b2e1e1a3 100644 --- a/src/freshdata/semantic/apply.py +++ b/src/freshdata/semantic/apply.py @@ -267,8 +267,28 @@ def run_semantic( from .backends import gather_proposals # noqa: PLC0415 - avoid import cycle from .consistency import run_consistency_checks # noqa: PLC0415 - avoid import cycle - ctx = build_semantic_context(df, config) - proposals = gather_proposals(df, ctx, config, memory=memory, profile=profile, report=report) + # The semantic context, experts and backends name columns by ``str(label)``. + # Map each name back to the frame's real label so non-string labels (from + # ``read_csv(header=None)`` or ``df.T``) can be indexed. Distinct labels + # that stringify alike (``1`` and ``"1"``) cannot be told apart by name, so + # they are left out of semantic repair rather than risk repairing the + # wrong column. + labels_by_name: dict[str, list[object]] = {} + for label in df.columns: + labels_by_name.setdefault(str(label), []).append(label) + ambiguous = sorted(name for name, labels in labels_by_name.items() if len(labels) > 1) + work = df + if ambiguous: + report.add_warning( + f"semantic repair skipped column(s) {ambiguous}: distinct column labels " + "share the same string name, so a repair could not be attributed to " + "one column. Rename the columns to enable semantic repair for them." + ) + work = df.loc[:, [str(label) not in ambiguous for label in df.columns]] + label_for = {str(label): label for label in work.columns} + + ctx = build_semantic_context(work, config) + proposals = gather_proposals(work, ctx, config, memory=memory, profile=profile, report=report) out = df if proposals: @@ -276,7 +296,8 @@ def run_semantic( if replacements: # Shallow copy so we never mutate the caller's frame when applying. out = df.copy(deep=False) - for col, mapping in replacements.items(): + for name, mapping in replacements.items(): + col = label_for.get(name, name) out[col] = _apply_column(out[col], mapping) # Cross-field checks route contradictions no single-column expert can see # (date ordering, unit conflicts, sensitive-column anomalies) to a human. diff --git a/src/freshdata/semantic/consistency.py b/src/freshdata/semantic/consistency.py index d38c18da..33d0cf3f 100644 --- a/src/freshdata/semantic/consistency.py +++ b/src/freshdata/semantic/consistency.py @@ -19,6 +19,7 @@ import warnings from collections.abc import Iterable, Mapping +import numpy as np import pandas as pd from ..config import CleanConfig @@ -60,10 +61,25 @@ def _rows_text(rows: Iterable[object]) -> str: return " ".join(named) +def _utc_naive(parsed: pd.Series) -> pd.Series: + """Express datetimes as naive UTC so a tz-aware column compares with a + naive one instead of raising; naive values are read as UTC.""" + if isinstance(parsed.dtype, pd.DatetimeTZDtype): + return parsed.dt.tz_convert("UTC").dt.tz_localize(None) + return parsed + + +def _utc_naive_timestamp(ts: pd.Timestamp) -> pd.Timestamp: + if ts.tzinfo is None: + return ts + return ts.tz_convert("UTC").tz_localize(None) + + def _as_datetime(series: pd.Series) -> pd.Series | None: - """Parse a column to datetimes, or ``None`` when it clearly is not one.""" + """Parse a column to naive-UTC datetimes, or ``None`` when it clearly is + not one.""" if pd.api.types.is_datetime64_any_dtype(series): - return series + return _utc_naive(series) if series.dtype != object: return None nonnull = series.dropna() @@ -73,11 +89,17 @@ def _as_datetime(series: pd.Series) -> pd.Series | None: warnings.simplefilter("ignore") try: parsed = pd.to_datetime(series, errors="coerce") + if not pd.api.types.is_datetime64_any_dtype(parsed): + # Mixed UTC offsets (or offset and naive strings) parse to an + # object column; ``utc=True`` puts every value on one clock. + parsed = pd.to_datetime(series, errors="coerce", utc=True) except (TypeError, ValueError, OverflowError): return None + if not pd.api.types.is_datetime64_any_dtype(parsed): + return None if parsed.notna().sum() / len(nonnull) < 0.6: return None - return parsed + return _utc_naive(parsed) def _modal_value(series: pd.Series) -> object: @@ -120,12 +142,18 @@ def _check_date_pair_ordering(df: pd.DataFrame, report: CleanReport) -> None: count = int(violations.sum()) if not 0 < count <= max(3, int(0.1 * n)): continue - start_modal = _modal_value(df[start_col][both]) - end_modal = _modal_value(df[end_col][both]) + # Work positionally: ``df.at[row, col]`` returns a Series when a + # row label is duplicated (e.g. after ``pd.concat``). + both_mask = both.to_numpy(dtype=bool) + start_raw = df[start_col] + end_raw = df[end_col] + start_modal = _modal_value(start_raw[both_mask]) + end_modal = _modal_value(end_raw[both_mask]) per_column: dict[str, list[object]] = {} - for row in df.index[violations]: - start_deviates = df.at[row, start_col] != start_modal - end_deviates = df.at[row, end_col] != end_modal + for pos in np.flatnonzero(violations.to_numpy(dtype=bool)): + row = df.index[pos] + start_deviates = start_raw.iloc[pos] != start_modal + end_deviates = end_raw.iloc[pos] != end_modal if end_deviates and not start_deviates: per_column.setdefault(str(end_col), []).append(row) elif start_deviates and not end_deviates: @@ -157,6 +185,8 @@ def _check_future_start_dates( reference_ts = pd.Timestamp(str(reference)) except (ValueError, TypeError): return + # Parsed columns are naive UTC (see ``_as_datetime``); compare on that clock. + reference_utc = _utc_naive_timestamp(reference_ts) for col in df.columns: if not _START_NAME.search(str(col)): continue @@ -164,7 +194,7 @@ def _check_future_start_dates( if parsed is None: continue n = int(parsed.notna().sum()) - future = parsed.notna() & (parsed > reference_ts) + future = parsed.notna() & (parsed > reference_utc) count = int(future.sum()) if n < 8 or not 0 < count <= max(3, int(0.1 * n)): continue @@ -195,9 +225,10 @@ def _check_policy_durations(df: pd.DataFrame, report: CleanReport) -> None: if str(repair_col) == str(retention_col): continue rows: list[object] = [] - for row in df.index: - retention = _duration_days(df.at[row, retention_col]) - repair = _duration_days(df.at[row, repair_col]) + pairs = zip(df.index, df[retention_col].tolist(), df[repair_col].tolist()) + for row, retention_value, repair_value in pairs: + retention = _duration_days(retention_value) + repair = _duration_days(repair_value) if retention is not None and repair is not None and repair < retention: rows.append(row) if not rows: @@ -253,20 +284,22 @@ def _check_fahrenheit_in_celsius(df: pd.DataFrame, report: CleanReport) -> None: if not _TEMP_NAME.search(str(col)): continue numeric = pd.to_numeric(df[col], errors="coerce") - nonnull = numeric.dropna() - if len(nonnull) < 8: + present = np.flatnonzero(numeric.notna().to_numpy(dtype=bool)) + if len(present) < 8: continue - flagged: list[object] = [] - for row in df.index[numeric.notna()]: - value = float(numeric.at[row]) - others = nonnull[nonnull.index != row] - if len(others) < 4 or value <= float(others.max()) + 10: - continue - converted = (value - 32.0) * 5.0 / 9.0 - if float(others.min()) - 0.5 <= converted <= float(others.max()) + 0.5: - flagged.append(row) - if not 0 < len(flagged) <= max(2, int(0.1 * len(nonnull))): + # Positional, so a duplicated row label cannot turn a cell into a + # Series. Only the column maximum can exceed every *other* value by + # more than 10 (a tied maximum never does), so it is the one candidate. + values = numeric.iloc[present].to_numpy(dtype=float) + top = int(np.argmax(values)) + value = float(values[top]) + others = np.delete(values, top) + if value <= float(others.max()) + 10: + continue + converted = (value - 32.0) * 5.0 / 9.0 + if not float(others.min()) - 0.5 <= converted <= float(others.max()) + 0.5: continue + flagged: list[object] = [df.index[present[top]]] report.add_warning( f"column '{col}': {len(flagged)} value(s) read as Fahrenheit in a " f"Celsius-ranged column (the converted value fits the column's " @@ -282,12 +315,14 @@ def _check_timezone_transitions(df: pd.DataFrame, report: CleanReport) -> None: if not _TZ_NAME.search(str(tz_col)): continue values = df[tz_col] - transitions = [ - row - for row in df.index[values.notna()] - if isinstance(values.at[row], str) - and _TZ_TRANSITION.search(values.at[row]) + # Positions, not labels: ``values.at[row]`` is a Series on a + # duplicated row label, which would silently hide the transition. + transition_positions = [ + pos + for pos, value in enumerate(values.tolist()) + if isinstance(value, str) and _TZ_TRANSITION.search(value) ] + transitions = [df.index[pos] for pos in transition_positions] n = int(values.notna().sum()) if n < 8 or not 0 < len(transitions) <= max(3, int(0.1 * n)): continue @@ -300,11 +335,12 @@ def _check_timezone_transitions(df: pd.DataFrame, report: CleanReport) -> None: for other in df.columns: if str(other) == str(tz_col): continue + other_values = df[other] window_rows = [ - row - for row in transitions - if isinstance(df.at[row, other], str) - and _WINDOW_VALUE.search(df.at[row, other]) + df.index[pos] + for pos in transition_positions + if isinstance(other_values.iloc[pos], str) + and _WINDOW_VALUE.search(other_values.iloc[pos]) ] if not window_rows: continue diff --git a/tests/test_semantic_label_and_tz_robustness.py b/tests/test_semantic_label_and_tz_robustness.py new file mode 100644 index 00000000..ef5a98df --- /dev/null +++ b/tests/test_semantic_label_and_tz_robustness.py @@ -0,0 +1,220 @@ +"""Semantic stage robustness: duplicate row labels, non-string column labels, +and tz-aware vs naive datetimes (#231, #232, #233 — semantic parts).""" + +from __future__ import annotations + +import warnings + +import pandas as pd + +import freshdata as fd +from freshdata.report import CleanReport +from freshdata.semantic.consistency import ( + _as_datetime, + _check_date_pair_ordering, + _check_fahrenheit_in_celsius, + _check_future_start_dates, + _check_policy_durations, + _check_timezone_transitions, +) + +DUP_INDEX = [0, 0, 1, 1, 2, 2, 3, 3, 4, 4] + + +def _report() -> CleanReport: + return CleanReport(rows_before=10, rows_after=10, cols_before=2, cols_after=2) + + +def _clean(df: pd.DataFrame, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fd.clean(df, verbose=False, return_report=True, **kwargs) + + +# -- #231 part 1: duplicate row index labels ----------------------------------- + + +def _ordering_frame(index=None) -> pd.DataFrame: + df = pd.DataFrame( + { + "start_date": pd.date_range("2024-01-01", periods=10).astype(str), + "end_date": pd.date_range("2024-02-01", periods=10).astype(str), + }, + index=index, + ) + df.iloc[3, 1] = "2023-01-01" + return df + + +def test_clean_with_duplicate_row_labels_reports_date_ordering_break(): + _, control = _clean(_ordering_frame().reset_index(drop=True), semantic_mode="auto") + _, report = _clean(_ordering_frame(DUP_INDEX), semantic_mode="auto") + + def ordering(rep): + return [w for w in rep.warnings if "break the date ordering" in w] + + assert ordering(control) + assert len(ordering(report)) == len(ordering(control)) + # Position 3 carries the original (duplicated) label 1. + assert all("(row 1)" in w for w in ordering(report)) + + +def test_date_pair_ordering_reports_original_labels_on_duplicate_index(): + report = _report() + _check_date_pair_ordering(_ordering_frame(DUP_INDEX), report) + assert report.warnings + assert all("(row 1)" in w for w in report.warnings) + + +def test_fahrenheit_check_handles_duplicate_row_labels(): + temps = [20.0, 21.0, 22.0, 23.0, 21.0, 71.6, 22.0, 20.0, 23.0, 21.5] + unique = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}), unique) + duplicated = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}, index=DUP_INDEX), duplicated) + + assert len(unique.warnings) == 1 and "(row 5)" in unique.warnings[0] + assert len(duplicated.warnings) == 1 and "(row 2)" in duplicated.warnings[0] + assert "1 value(s)" in duplicated.warnings[0] + + +def test_fahrenheit_check_ignores_a_tied_maximum(): + temps = [20.0, 21.0, 22.0, 23.0, 21.0, 71.6, 71.6, 20.0, 23.0, 21.5] + report = _report() + _check_fahrenheit_in_celsius(pd.DataFrame({"temp_c": temps}, index=DUP_INDEX), report) + assert report.warnings == [] + + +def test_timezone_transition_found_on_duplicate_row_labels(): + df = pd.DataFrame( + { + "timezone": ["UTC"] * 9 + ["UTC→IST"], + "window": ["09:00-17:00"] * 10, + }, + index=DUP_INDEX, + ) + report = _report() + _check_timezone_transitions(df, report) + assert any("column 'timezone'" in w and "(row 4)" in w for w in report.warnings) + assert any("column 'window'" in w and "(row 4)" in w for w in report.warnings) + + +def test_policy_duration_conflict_found_on_duplicate_row_labels(): + df = pd.DataFrame( + {"retention": ["7 years", "1 year"], "purge_after": ["30 days", "2 years"]}, + index=[7, 7], + ) + report = _report() + _check_policy_durations(df, report) + assert len(report.warnings) == 2 + assert all("1 row(s) (row 7)" in w for w in report.warnings) + + +# -- #232 part 1: non-string column labels ------------------------------------- + + +def test_semantic_clean_accepts_integer_column_labels(): + df = pd.DataFrame({1: ["yes", "no", "yes", "no"], 2: ["twenty", "5", "6", "7"]}) + out, report = _clean(df, semantic_mode="auto", column_names=False) + + assert out[2].tolist() == [20, 5, 6, 7] + assert "2" not in out.columns + semantic = [a for a in report.actions if a.step == "semantic"] + assert any(a.column == "2" and a.description.startswith("Normalized") for a in semantic) + + +def test_semantic_repair_skips_labels_that_stringify_alike(): + df = pd.DataFrame({1: ["a", "b", "c", "d"], "1": ["twenty", "5", "6", "7"]}) + out, report = _clean(df, semantic_mode="auto", column_names=False) + + assert list(out.columns)[:2] == [1, "1"] + assert out["1"].tolist() == ["twenty", "5", "6", "7"] + assert any("semantic repair skipped column(s) ['1']" in w for w in report.warnings) + assert not [a for a in report.actions if a.step == "semantic" and a.column == "1"] + + +def test_semantic_clean_with_colliding_labels_still_repairs_other_columns(): + df = pd.DataFrame( + {1: ["a", "b", "c", "d"], "1": ["x", "y", "z", "w"], "qty": ["twenty", "5", "6", "7"]} + ) + out, _ = _clean(df, semantic_mode="auto", column_names=False) + assert out["qty"].tolist() == [20, 5, 6, 7] + + +# -- #233 part 1: tz-aware vs naive datetimes ---------------------------------- + + +def test_clean_with_tz_aware_start_and_naive_end_completes(): + df = pd.DataFrame( + { + "start_date": pd.date_range("2024-01-01", periods=10, tz="UTC"), + "end_date": pd.date_range("2024-02-01", periods=10), + } + ) + _clean(df, semantic_mode="auto") + + +def test_date_pair_ordering_compares_aware_and_naive_on_utc(): + start = pd.Series(pd.date_range("2024-01-01", periods=10, tz="UTC")) + end = pd.Series(pd.date_range("2024-02-01", periods=10)) + end.iloc[5] = pd.Timestamp("2023-12-01") + report = _report() + _check_date_pair_ordering(pd.DataFrame({"start_date": start, "end_date": end}), report) + assert report.warnings + assert all("(row 5)" in w for w in report.warnings) + + +def test_as_datetime_normalizes_offsets_to_naive_utc(): + aware = pd.Series(["2024-01-01T10:00:00+05:00"] * 5) + mixed = pd.Series(["2024-03-10T01:30:00-05:00", "2024-03-10T03:30:00-04:00"] * 3) + tz_column = pd.Series(pd.date_range("2024-01-01", periods=5, tz="Asia/Kolkata")) + + parsed_aware = _as_datetime(aware) + parsed_mixed = _as_datetime(mixed) + parsed_tz = _as_datetime(tz_column) + + assert parsed_aware is not None and parsed_aware.dt.tz is None + assert parsed_aware.iloc[0] == pd.Timestamp("2024-01-01 05:00:00") + assert parsed_mixed is not None and parsed_mixed.dt.tz is None + assert parsed_mixed.iloc[0] == pd.Timestamp("2024-03-10 06:30:00") + assert parsed_mixed.iloc[1] == pd.Timestamp("2024-03-10 07:30:00") + assert parsed_tz is not None and parsed_tz.dt.tz is None + assert parsed_tz.iloc[0] == pd.Timestamp("2023-12-31 18:30:00") + + +def test_date_pair_ordering_with_mixed_offset_strings(): + starts = [f"2024-03-{d:02d}T08:00:00-05:00" for d in range(1, 11)] + ends = [f"2024-03-{d:02d}T10:00:00-04:00" for d in range(1, 11)] + # 08:00-05:00 is 13:00 UTC; 12:00-04:00 would be 16:00 UTC (ordered), but + # 07:00-04:00 is 11:00 UTC, before its start. + ends[4] = "2024-03-05T07:00:00-04:00" + report = _report() + _check_date_pair_ordering(pd.DataFrame({"start_date": starts, "end_date": ends}), report) + assert report.warnings + assert all("(row 4)" in w for w in report.warnings) + + +def test_future_start_dates_with_tz_aware_column_and_naive_reference(): + starts = pd.Series(pd.date_range("2024-01-01", periods=10, tz="UTC")) + starts.iloc[7] = pd.Timestamp("2025-01-01", tz="UTC") + report = _report() + _check_future_start_dates( + pd.DataFrame({"start_date": starts}), {"reference_date": "2024-06-01"}, report + ) + assert len(report.warnings) == 1 and "(row 7)" in report.warnings[0] + assert "2024-06-01" in report.warnings[0] + + +def test_future_start_dates_with_naive_column_and_aware_reference(): + starts = pd.Series(pd.date_range("2024-05-25", periods=10)) + report = _report() + # 2024-06-01T00:00+05:30 is 2024-05-31 18:30 UTC, so only 2024-06-01 onward + # (positions 7-9) lies after it. + _check_future_start_dates( + pd.DataFrame({"start_date": starts}), + {"reference_date": "2024-06-01T00:00:00+05:30"}, + report, + ) + assert len(report.warnings) == 1 + assert "3 date(s)" in report.warnings[0] + assert "(row 7) (row 8) (row 9)" in report.warnings[0]