Skip to content

Commit ec5fdaf

Browse files
committed
Compare upsert rows column by column instead of cell by cell
1 parent aafad6e commit ec5fdaf

2 files changed

Lines changed: 62 additions & 20 deletions

File tree

‎pyiceberg/table/upsert_util.py‎

Lines changed: 33 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,25 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool:
5353
return len(df.select(join_cols).group_by(join_cols).aggregate([([], "count_all")]).filter(pc.field("count_all") > 1)) > 0
5454

5555

56+
def _values_differ(
57+
source_values: pa.Array | pa.ChunkedArray, target_values: pa.Array | pa.ChunkedArray
58+
) -> pa.Array | pa.ChunkedArray:
59+
"""Return a mask of the positions where the values are not equal, treating two nulls as equal."""
60+
try:
61+
equal = pc.fill_null(pc.equal(source_values, target_values), False)
62+
except (pa.ArrowNotImplementedError, pa.ArrowInvalid, pa.ArrowTypeError):
63+
# PyArrow has no comparison kernel for nested types
64+
source_list = source_values.to_pylist()
65+
target_list = target_values.to_pylist()
66+
return pa.array(
67+
[source_value != target_value for source_value, target_value in zip(source_list, target_list, strict=True)],
68+
type=pa.bool_(),
69+
)
70+
71+
both_null = pc.and_(pc.is_null(source_values), pc.is_null(target_values))
72+
return pc.invert(pc.or_(equal, both_null))
73+
74+
5675
def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table:
5776
"""
5877
Return a table with rows that need to be updated in the target table based on the join columns.
@@ -72,7 +91,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
7291
# When the target table is empty, there is nothing to update :)
7392
return source_table.schema.empty_table()
7493

75-
# We need to compare non_key_cols in Python as PyArrow
94+
# We need to compare non_key_cols without a join as PyArrow
7695
# 1. Cannot do a join when non-join columns have complex types
7796
# 2. Cannot compare columns with complex types
7897
# See: https://github.com/apache/arrow/issues/35785
@@ -100,25 +119,20 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
100119
# Step 3: Perform an inner join to find which rows from source exist in target
101120
matching_indices = source_index.join(target_index, keys=list(join_cols_set), join_type="inner")
102121

103-
# Step 4: Compare all rows using Python
104-
to_update_indices = []
105-
for source_idx, target_idx in zip(
106-
matching_indices[SOURCE_INDEX_COLUMN_NAME].to_pylist(),
107-
matching_indices[TARGET_INDEX_COLUMN_NAME].to_pylist(),
108-
strict=True,
109-
):
110-
source_row = source_table.slice(source_idx, 1)
111-
target_row = target_table.slice(target_idx, 1)
112-
113-
for key in non_key_cols:
114-
source_val = source_row.column(key)[0].as_py()
115-
target_val = target_row.column(key)[0].as_py()
116-
if source_val != target_val:
117-
to_update_indices.append(source_idx)
118-
break
122+
# Step 4: Compare the matched rows column by column
123+
source_indices = matching_indices[SOURCE_INDEX_COLUMN_NAME]
124+
target_indices = matching_indices[TARGET_INDEX_COLUMN_NAME]
125+
126+
changed: pa.Array | pa.ChunkedArray | None = None
127+
for key in non_key_cols:
128+
column_changed = _values_differ(
129+
source_table.column(key).take(source_indices),
130+
target_table.column(key).take(target_indices),
131+
)
132+
changed = column_changed if changed is None else pc.or_(changed, column_changed)
119133

120134
# Step 5: Take rows from source table using the indices and cast to target schema
121-
if to_update_indices:
122-
return source_table.take(to_update_indices)
135+
if changed is not None and pc.any(changed).as_py():
136+
return source_table.take(pc.filter(source_indices, changed))
123137
else:
124138
return source_table.schema.empty_table()

‎tests/table/test_upsert.py‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
from pyiceberg.schema import Schema
3232
from pyiceberg.table import Table, UpsertResult
3333
from pyiceberg.table.snapshots import Operation
34-
from pyiceberg.table.upsert_util import create_match_filter
34+
from pyiceberg.table.upsert_util import create_match_filter, get_rows_to_update
3535
from pyiceberg.transforms import DayTransform
3636
from pyiceberg.types import IntegerType, NestedField, StringType, StructType, TimestampType
3737
from tests.catalog.test_base import InMemoryCatalog
@@ -927,3 +927,31 @@ def test_upsert_snapshot_properties(catalog: Catalog) -> None:
927927
for snapshot in snapshots[initial_snapshot_count:]:
928928
assert snapshot.summary is not None
929929
assert snapshot.summary.additional_properties.get("test_prop") == "test_value"
930+
931+
932+
def test_get_rows_to_update_returns_changed_rows() -> None:
933+
source = pa.table({"pk": [1, 2, 3], "value": [10, 20, 30]})
934+
target = pa.table({"pk": [1, 2, 3], "value": [10, 21, 30]})
935+
936+
assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [20]}
937+
938+
939+
def test_get_rows_to_update_returns_a_row_once() -> None:
940+
source = pa.table({"pk": [1], "a": [1], "b": [2]})
941+
target = pa.table({"pk": [1], "a": [9], "b": [9]})
942+
943+
assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [1], "a": [1], "b": [2]}
944+
945+
946+
def test_get_rows_to_update_treats_nulls_as_equal() -> None:
947+
source = pa.table({"pk": [1, 2], "value": pa.array([None, 1], type=pa.int64())})
948+
target = pa.table({"pk": [1, 2], "value": pa.array([None, 2], type=pa.int64())})
949+
950+
assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [1]}
951+
952+
953+
def test_get_rows_to_update_compares_nested_values() -> None:
954+
source = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 2}])})
955+
target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])})
956+
957+
assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]}

0 commit comments

Comments
 (0)