diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 1a5692fa69..fb2266d83e 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -53,6 +53,18 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool: return len(df.select(join_cols).group_by(join_cols).aggregate([([], "count_all")]).filter(pc.field("count_all") > 1)) > 0 +def _struct_values_differ(source_values: pa.ChunkedArray, target_values: pa.ChunkedArray) -> pa.ChunkedArray: + """Compare two struct columns field by field, as PyArrow cannot compare the structs themselves.""" + # A null struct has null fields, so the struct itself decides those rows + changed = pc.not_equal(pc.is_null(source_values), pc.is_null(target_values)) + for field in source_values.type: + changed = pc.or_( + changed, + _values_differ(pc.struct_field(source_values, field.name), pc.struct_field(target_values, field.name)), + ) + return changed + + def _values_differ( source_values: pa.Array | pa.ChunkedArray, target_values: pa.Array | pa.ChunkedArray ) -> pa.Array | pa.ChunkedArray: @@ -61,6 +73,14 @@ def _values_differ( equal = pc.fill_null(pc.equal(source_values, target_values), False) except (pa.ArrowNotImplementedError, pa.ArrowInvalid, pa.ArrowTypeError): # PyArrow has no comparison kernel for nested types + if pa.types.is_struct(source_values.type) and source_values.type == target_values.type: + return _struct_values_differ(source_values, target_values) + + # Different types: cast the source to the target type and compare again, e.g. a naive against + # a zoned timestamp, or a struct of string against the struct of large_string a scan returns + if source_values.type != target_values.type: + return _values_differ(source_values.cast(target_values.type), target_values) + source_list = source_values.to_pylist() target_list = target_values.to_pylist() return pa.array( diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..4afce79e35 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -955,3 +955,102 @@ def test_get_rows_to_update_compares_nested_values() -> None: target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])}) assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]} + + +def test_get_rows_to_update_compares_columns_of_a_different_type() -> None: + """A naive and a zoned timestamp of the same instant are not a change.""" + target = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 2], pa.timestamp("us", tz="UTC"))}) + source = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 9], pa.timestamp("us"))}) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2] + + +def test_get_rows_to_update_compares_a_struct_that_is_null() -> None: + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field("nested", pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])), + ] + ) + target = pa.Table.from_pylist( + [ + {"id": 1, "nested": None}, + {"id": 2, "nested": None}, + {"id": 3, "nested": {"a": 1, "b": "x"}}, + {"id": 4, "nested": {"a": 1, "b": "x"}}, + {"id": 5, "nested": {"a": 1, "b": None}}, + {"id": 6, "nested": {"a": None, "b": None}}, + {"id": 7, "nested": None}, + ], + schema=schema, + ) + source = pa.Table.from_pylist( + [ + {"id": 1, "nested": None}, # unchanged + {"id": 2, "nested": {"a": 1, "b": "x"}}, # null to a struct + {"id": 3, "nested": None}, # a struct to null + {"id": 4, "nested": {"a": 1, "b": "y"}}, # one field differs + {"id": 5, "nested": {"a": 1, "b": None}}, # unchanged, with a null field + {"id": 6, "nested": None}, # a struct of null fields to null + {"id": 7, "nested": {"a": None, "b": None}}, # null to a struct of null fields + ], + schema=schema, + ) + + assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4, 6, 7] + + +def test_get_rows_to_update_compares_a_struct_whose_fields_are_in_another_order() -> None: + source_type = pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.string())]) + target_type = pa.struct([pa.field("b", pa.string()), pa.field("a", pa.int64())]) + source = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 2, "b": "y"}], type=source_type)}) + target = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 9, "b": "y"}], type=target_type)}) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2] + + +def test_get_rows_to_update_compares_a_nested_struct() -> None: + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field("outer", pa.struct([pa.field("inner", pa.struct([pa.field("value", pa.int32())]))])), + ] + ) + target = pa.Table.from_pylist( + [ + {"id": 1, "outer": {"inner": {"value": 1}}}, + {"id": 2, "outer": {"inner": {"value": 1}}}, + {"id": 3, "outer": {"inner": None}}, + {"id": 4, "outer": {"inner": {"value": 1}}}, + ], + schema=schema, + ) + source = pa.Table.from_pylist( + [ + {"id": 1, "outer": {"inner": {"value": 1}}}, # unchanged + {"id": 2, "outer": {"inner": {"value": 2}}}, # the innermost value differs + {"id": 3, "outer": {"inner": {"value": 1}}}, # a null inner struct to a struct + {"id": 4, "outer": {"inner": None}}, # a struct to a null inner struct + ], + schema=schema, + ) + + assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4] + + +def test_get_rows_to_update_compares_a_struct_read_as_large_string() -> None: + """A scan reads the strings of a struct as large_string, while a dataframe holds them as string.""" + source = pa.table( + { + "id": pa.array([1, 2], pa.int32()), + "nested": pa.array([{"a": "x"}, {"a": "y"}], type=pa.struct([pa.field("a", pa.string())])), + } + ) + target = pa.table( + { + "id": pa.array([1, 2], pa.int32()), + "nested": pa.array([{"a": "x"}, {"a": "z"}], type=pa.struct([pa.field("a", pa.large_string())])), + } + ) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]