From 517bc2d7c1c9cec2f1fa89d1d4aff82d3d9dd58a Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Fri, 9 Oct 2026 15:39:26 -0500 Subject: [PATCH] Fix split.dictionary defaults for invalid inputs and null values --- tests/recipes/wrangles/test_split.py | 158 +++++++++++++++++++++++++++ wrangles/recipe_wrangles/split.py | 48 ++++++-- 2 files changed, 194 insertions(+), 12 deletions(-) diff --git a/tests/recipes/wrangles/test_split.py b/tests/recipes/wrangles/test_split.py index 509f9df8..c59267f6 100644 --- a/tests/recipes/wrangles/test_split.py +++ b/tests/recipes/wrangles/test_split.py @@ -905,6 +905,164 @@ def test_split_dictionary_default(self): df['Col4'][0] == 'D' ) + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + @pytest.mark.parametrize("invalid", [ + "Timed Out", "", None, float("nan"), 123, False, [], + "{invalid JSON}", '["not a dictionary"]', "null", + ]) + def test_split_dictionary_invalid_input_uses_default(self, invalid, output_format): + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", + "default": {"Voltage": {}}, + "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({"atts": [ + {"Voltage": {"value": 120, "uom": "VAC"}}, + invalid, + '{"Voltage": {"value": 12, "uom": "VDC"}}', + ]}), + ) + expected = [{"value": 120, "uom": "VAC"}, {}, {"value": 12, "uom": "VDC"}] + if output_format == "columns": + assert df["Voltage"].tolist() == expected + else: + assert df["Keys"].tolist() == [["Voltage"]] * 3 + assert df["Values"].tolist() == [[value] for value in expected] + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + @pytest.mark.parametrize("invalid", ["Timed Out", None, "{invalid JSON}", []]) + def test_split_dictionary_invalid_input_without_default_raises(self, invalid, output_format): + with pytest.raises(ValueError, match="is not a valid Dictionary"): + wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({"atts": [{"Voltage": 120}, invalid]}), + ) + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + def test_split_dictionary_explicit_empty_default(self, output_format): + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "default": {}, "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({"atts": ["Timed Out", None]}), + ) + if output_format == "columns": + assert df.columns.tolist() == ["atts"] + else: + assert df["Keys"].tolist() == [[], []] + assert df["Values"].tolist() == [[], []] + + @pytest.mark.parametrize("invalid_default", ["Timed Out", 123, []]) + def test_split_dictionary_invalid_default_raises(self, invalid_default): + with pytest.raises(ValueError, match="is not a valid Dictionary"): + wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "default": invalid_default, + }}]}, + dataframe=pd.DataFrame({"atts": [{"Voltage": 120}]}), + ) + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + @pytest.mark.parametrize("default", [{}, "", [], "fallback", 0, False]) + def test_split_dictionary_json_null_uses_default(self, default, output_format): + df = wrangles.recipe.run( + {"wrangles": [ + {"convert.from_json": {"input": "Source", "output": "atts"}}, + {"split.dictionary": { + "input": "atts", "default": {"Voltage": default}, + "output_format": output_format, + }}, + ]}, + dataframe=pd.DataFrame({"Source": [ + '{"Voltage": {"value": 120, "uom": "VAC"}}', + '{"Voltage": {"value": 120, "uom": "VAC"}}', + '{"Voltage": null}', + '{}', + ]}), + ) + expected = [{"value": 120, "uom": "VAC"}] * 2 + [default, default] + if output_format == "columns": + assert df["Voltage"].tolist() == expected + else: + assert df["Keys"].tolist() == [["Voltage"]] * 4 + assert df["Values"].tolist() == [[value] for value in expected] + assert df.at[2, "atts"] == {"Voltage": None} + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + def test_split_dictionary_defaults_preserve_existing_values(self, output_format): + values = { + "Zero": 0, "False": False, "Blank": "", "List": [], "Dictionary": {}, + "Nested": {"child": None}, "Unconfigured": None, "Voltage": 120, + } + defaults = {key: "fallback" for key in values if key not in ("Unconfigured", "Voltage")} + defaults["voltage"] = {} + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "default": defaults, "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({"atts": [values]}), + ) + expected = {**values, "voltage": {}} + if output_format == "columns": + expected["Unconfigured"] = "" + assert df[list(expected)].to_dict("records") == [expected] + else: + assert dict(zip(df.at[0, "Keys"], df.at[0, "Values"])) == expected + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + def test_split_dictionary_defaults_after_multiple_inputs(self, output_format): + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": ["First", "Second"], + "default": {"Voltage": {}, "Missing": "fallback"}, + "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({ + "First": [{"Voltage": 1}, "Timed Out", {"Voltage": 1}, {"Voltage": None}], + "Second": ["Timed Out", {"Voltage": 2}, {"Voltage": None}, {"Voltage": 3}], + }), + ) + expected = [1, 2, {}, 3] + if output_format == "columns": + assert df["Voltage"].tolist() == expected + assert df["Missing"].tolist() == ["fallback"] * 4 + else: + assert df["Keys"].tolist() == [["Voltage", "Missing"]] * 4 + assert df["Values"].tolist() == [[value, "fallback"] for value in expected] + + @pytest.mark.parametrize("output_format", ["columns", "to_lists"]) + def test_split_dictionary_mutable_defaults_are_independent(self, output_format): + default = {"Voltage": {"values": []}} + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "default": default, "output_format": output_format, + }}]}, + dataframe=pd.DataFrame({"atts": [{"Voltage": None}, {}, "Timed Out"]}), + ) + cells = df["Voltage"].tolist() if output_format == "columns" else [ + row[0] for row in df["Values"] + ] + cells[0]["values"].append(120) + assert cells[1:] == [{"values": []}, {"values": []}] + assert default == {"Voltage": {"values": []}} + + def test_split_dictionary_defaults_with_where_and_renaming(self): + df = wrangles.recipe.run( + {"wrangles": [{"split.dictionary": { + "input": "atts", "output": [{"Voltage": "Supply Voltage"}], + "default": {"Voltage": {}}, "where": "Process > 0", + }}]}, + dataframe=pd.DataFrame({ + "atts": [{"Voltage": None}, "Timed Out", {"Voltage": None}], + "Supply Voltage": ["old null", "old error", "keep"], + "Process": [1, 1, 0], + }), + ) + assert df["Supply Voltage"].tolist() == [{}, {}, "keep"] + def test_split_dictionary_multiple(self): """ Test splitting a list of dictionaries diff --git a/wrangles/recipe_wrangles/split.py b/wrangles/recipe_wrangles/split.py index 01f18cd0..63d59d93 100644 --- a/wrangles/recipe_wrangles/split.py +++ b/wrangles/recipe_wrangles/split.py @@ -3,6 +3,7 @@ """ # Rename List to _list to be able to use function name list without clashing from typing import Union as _Union, List as _list +from copy import deepcopy as _deepcopy import logging as _logging import pandas as _pd from .. import format as _format @@ -25,7 +26,8 @@ def dictionary( description: |- Split one or more dictionaries into columns. The dictionary keys will be returned as the new column headers. - If the dictionaries contain overlapping values, the last value will be returned. + Dictionary keys are case-sensitive. With multiple input columns, + later dictionaries overwrite earlier keys before defaults are applied. additionalProperties: false required: - input @@ -36,9 +38,9 @@ def dictionary( - integer - array description: |- - Name or lists of the column(s) containing dictionaries to be split. - If providing multiple dictionaries and the dictionaries - contain overlapping values, the last value will be returned. + Name or list of columns containing dictionaries or JSON object strings to split. + For overlapping keys, later input columns take precedence + before defaults are applied. output: type: - string @@ -55,8 +57,15 @@ def dictionary( default: type: object description: >- - Provide a set of default headings and values - if they are not found within the input + Provide defaults for missing top-level keys or keys with null values + after merging the input dictionaries. Keys use their original names + before output renaming. Non-null values, including 0, false, empty + strings, lists, and dictionaries, are preserved. Nested values are + not filled recursively. + When default is supplied (including {}), invalid or non-dictionary + input cells are treated as empty dictionaries, so valid values from + other input columns are retained. Without default, invalid input + raises an error. output_format: type: string enum: @@ -65,11 +74,14 @@ def dictionary( description: |- How to split the dictionary. columns creates one output column for each dictionary key. + In recipes, remaining null cells are normalized to empty strings. to_lists creates two output columns containing lists of keys and values. + Remaining null values inside these lists are preserved. """ if output_format not in ["columns", "to_lists"]: raise ValueError("output_format must be one of: columns, to_lists") + has_default = default is not None if default is None: default = {} _logging.debug(f": Splitting dictionaries :: input :: {input}") @@ -88,12 +100,24 @@ def _parse_dict_or_json(val): raise ValueError(f'{val} is not a valid Dictionary') from None - # Merge each row's dictionaries so duplicate keys follow existing behavior: - # later input columns overwrite earlier input columns. - dicts = [ - dict(_itertools.chain.from_iterable(_parse_dict_or_json(d) for d in ([default] + row.tolist()))) - for row in df[input].values - ] + # Validate defaults separately so invalid configuration cannot use fallback. + default = dict(_parse_dict_or_json(default)) + dicts = [] + for row in df[input].values: + # Keep default keys first, preserving the existing output column order. + merged = dict.fromkeys(default) + for value in row: + try: + merged.update(_parse_dict_or_json(value)) + except ValueError: + if not has_default: + raise + + # Apply defaults after merging so later input columns still take precedence. + for key, value in default.items(): + if merged[key] is None: + merged[key] = _deepcopy(value) + dicts.append(merged) if output_format == "to_lists": if output is None: