From 197beb57f93df40b2997ff516a33dbec0ea2c735 Mon Sep 17 00:00:00 2001 From: Gaspard Merten Date: Wed, 11 Nov 2026 22:23:25 +0100 Subject: [PATCH] Compare struct upsert columns field by field in PyArrow Since #4334 the upsert comparison runs column by column in PyArrow, but struct columns still fall back to Python, a dict per row. Structs are now compared field by field, recursing into nested structs. A scan returns the strings of a struct as large_string while a dataframe holds string, so the two struct types differ in every real upsert. When PyArrow cannot compare two columns of different types, the source column is cast to the target type first, which lets those structs take the PyArrow path. Upserting 30 struct columns over 200k rows goes from 5.5s to 2.9s. --- pyiceberg/table/upsert_util.py | 20 +++++++ tests/table/test_upsert.py | 99 ++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+) diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 1a5692fa69..fb2266d83e 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -53,6 +53,18 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool: return len(df.select(join_cols).group_by(join_cols).aggregate([([], "count_all")]).filter(pc.field("count_all") > 1)) > 0 +def _struct_values_differ(source_values: pa.ChunkedArray, target_values: pa.ChunkedArray) -> pa.ChunkedArray: + """Compare two struct columns field by field, as PyArrow cannot compare the structs themselves.""" + # A null struct has null fields, so the struct itself decides those rows + changed = pc.not_equal(pc.is_null(source_values), pc.is_null(target_values)) + for field in source_values.type: + changed = pc.or_( + changed, + _values_differ(pc.struct_field(source_values, field.name), pc.struct_field(target_values, field.name)), + ) + return changed + + def _values_differ( source_values: pa.Array | pa.ChunkedArray, target_values: pa.Array | pa.ChunkedArray ) -> pa.Array | pa.ChunkedArray: @@ -61,6 +73,14 @@ def _values_differ( equal = pc.fill_null(pc.equal(source_values, target_values), False) except (pa.ArrowNotImplementedError, pa.ArrowInvalid, pa.ArrowTypeError): # PyArrow has no comparison kernel for nested types + if pa.types.is_struct(source_values.type) and source_values.type == target_values.type: + return _struct_values_differ(source_values, target_values) + + # Different types: cast the source to the target type and compare again, e.g. a naive against + # a zoned timestamp, or a struct of string against the struct of large_string a scan returns + if source_values.type != target_values.type: + return _values_differ(source_values.cast(target_values.type), target_values) + source_list = source_values.to_pylist() target_list = target_values.to_pylist() return pa.array( diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..4afce79e35 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -955,3 +955,102 @@ def test_get_rows_to_update_compares_nested_values() -> None: target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])}) assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]} + + +def test_get_rows_to_update_compares_columns_of_a_different_type() -> None: + """A naive and a zoned timestamp of the same instant are not a change.""" + target = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 2], pa.timestamp("us", tz="UTC"))}) + source = pa.table({"id": pa.array([1, 2], pa.int32()), "ts": pa.array([1, 9], pa.timestamp("us"))}) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2] + + +def test_get_rows_to_update_compares_a_struct_that_is_null() -> None: + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field("nested", pa.struct([pa.field("a", pa.int32()), pa.field("b", pa.string())])), + ] + ) + target = pa.Table.from_pylist( + [ + {"id": 1, "nested": None}, + {"id": 2, "nested": None}, + {"id": 3, "nested": {"a": 1, "b": "x"}}, + {"id": 4, "nested": {"a": 1, "b": "x"}}, + {"id": 5, "nested": {"a": 1, "b": None}}, + {"id": 6, "nested": {"a": None, "b": None}}, + {"id": 7, "nested": None}, + ], + schema=schema, + ) + source = pa.Table.from_pylist( + [ + {"id": 1, "nested": None}, # unchanged + {"id": 2, "nested": {"a": 1, "b": "x"}}, # null to a struct + {"id": 3, "nested": None}, # a struct to null + {"id": 4, "nested": {"a": 1, "b": "y"}}, # one field differs + {"id": 5, "nested": {"a": 1, "b": None}}, # unchanged, with a null field + {"id": 6, "nested": None}, # a struct of null fields to null + {"id": 7, "nested": {"a": None, "b": None}}, # null to a struct of null fields + ], + schema=schema, + ) + + assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4, 6, 7] + + +def test_get_rows_to_update_compares_a_struct_whose_fields_are_in_another_order() -> None: + source_type = pa.struct([pa.field("a", pa.int64()), pa.field("b", pa.string())]) + target_type = pa.struct([pa.field("b", pa.string()), pa.field("a", pa.int64())]) + source = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 2, "b": "y"}], type=source_type)}) + target = pa.table({"id": pa.array([1, 2]), "nested": pa.array([{"a": 1, "b": "x"}, {"a": 9, "b": "y"}], type=target_type)}) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2] + + +def test_get_rows_to_update_compares_a_nested_struct() -> None: + schema = pa.schema( + [ + pa.field("id", pa.int32()), + pa.field("outer", pa.struct([pa.field("inner", pa.struct([pa.field("value", pa.int32())]))])), + ] + ) + target = pa.Table.from_pylist( + [ + {"id": 1, "outer": {"inner": {"value": 1}}}, + {"id": 2, "outer": {"inner": {"value": 1}}}, + {"id": 3, "outer": {"inner": None}}, + {"id": 4, "outer": {"inner": {"value": 1}}}, + ], + schema=schema, + ) + source = pa.Table.from_pylist( + [ + {"id": 1, "outer": {"inner": {"value": 1}}}, # unchanged + {"id": 2, "outer": {"inner": {"value": 2}}}, # the innermost value differs + {"id": 3, "outer": {"inner": {"value": 1}}}, # a null inner struct to a struct + {"id": 4, "outer": {"inner": None}}, # a struct to a null inner struct + ], + schema=schema, + ) + + assert get_rows_to_update(source, target, ["id"]).sort_by("id").column("id").to_pylist() == [2, 3, 4] + + +def test_get_rows_to_update_compares_a_struct_read_as_large_string() -> None: + """A scan reads the strings of a struct as large_string, while a dataframe holds them as string.""" + source = pa.table( + { + "id": pa.array([1, 2], pa.int32()), + "nested": pa.array([{"a": "x"}, {"a": "y"}], type=pa.struct([pa.field("a", pa.string())])), + } + ) + target = pa.table( + { + "id": pa.array([1, 2], pa.int32()), + "nested": pa.array([{"a": "x"}, {"a": "z"}], type=pa.struct([pa.field("a", pa.large_string())])), + } + ) + + assert get_rows_to_update(source, target, ["id"]).column("id").to_pylist() == [2]