@@ -53,6 +53,25 @@ def has_duplicate_rows(df: pyarrow_table, join_cols: list[str]) -> bool:
5353 return len (df .select (join_cols ).group_by (join_cols ).aggregate ([([], "count_all" )]).filter (pc .field ("count_all" ) > 1 )) > 0
5454
5555
56+ def _values_differ (
57+ source_values : pa .Array | pa .ChunkedArray , target_values : pa .Array | pa .ChunkedArray
58+ ) -> pa .Array | pa .ChunkedArray :
59+ """Return a mask of the positions where the values are not equal, treating two nulls as equal."""
60+ try :
61+ equal = pc .fill_null (pc .equal (source_values , target_values ), False )
62+ except (pa .ArrowNotImplementedError , pa .ArrowInvalid , pa .ArrowTypeError ):
63+ # PyArrow has no comparison kernel for nested types
64+ source_list = source_values .to_pylist ()
65+ target_list = target_values .to_pylist ()
66+ return pa .array (
67+ [source_value != target_value for source_value , target_value in zip (source_list , target_list , strict = True )],
68+ type = pa .bool_ (),
69+ )
70+
71+ both_null = pc .and_ (pc .is_null (source_values ), pc .is_null (target_values ))
72+ return pc .invert (pc .or_ (equal , both_null ))
73+
74+
5675def get_rows_to_update (source_table : pa .Table , target_table : pa .Table , join_cols : list [str ]) -> pa .Table :
5776 """
5877 Return a table with rows that need to be updated in the target table based on the join columns.
@@ -72,7 +91,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
7291 # When the target table is empty, there is nothing to update :)
7392 return source_table .schema .empty_table ()
7493
75- # We need to compare non_key_cols in Python as PyArrow
94+ # We need to compare non_key_cols without a join as PyArrow
7695 # 1. Cannot do a join when non-join columns have complex types
7796 # 2. Cannot compare columns with complex types
7897 # See: https://github.com/apache/arrow/issues/35785
@@ -100,25 +119,20 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
100119 # Step 3: Perform an inner join to find which rows from source exist in target
101120 matching_indices = source_index .join (target_index , keys = list (join_cols_set ), join_type = "inner" )
102121
103- # Step 4: Compare all rows using Python
104- to_update_indices = []
105- for source_idx , target_idx in zip (
106- matching_indices [SOURCE_INDEX_COLUMN_NAME ].to_pylist (),
107- matching_indices [TARGET_INDEX_COLUMN_NAME ].to_pylist (),
108- strict = True ,
109- ):
110- source_row = source_table .slice (source_idx , 1 )
111- target_row = target_table .slice (target_idx , 1 )
112-
113- for key in non_key_cols :
114- source_val = source_row .column (key )[0 ].as_py ()
115- target_val = target_row .column (key )[0 ].as_py ()
116- if source_val != target_val :
117- to_update_indices .append (source_idx )
118- break
122+ # Step 4: Compare the matched rows column by column
123+ source_indices = matching_indices [SOURCE_INDEX_COLUMN_NAME ]
124+ target_indices = matching_indices [TARGET_INDEX_COLUMN_NAME ]
125+
126+ changed : pa .Array | pa .ChunkedArray | None = None
127+ for key in non_key_cols :
128+ column_changed = _values_differ (
129+ source_table .column (key ).take (source_indices ),
130+ target_table .column (key ).take (target_indices ),
131+ )
132+ changed = column_changed if changed is None else pc .or_ (changed , column_changed )
119133
120134 # Step 5: Take rows from source table using the indices and cast to target schema
121- if to_update_indices :
122- return source_table .take (to_update_indices )
135+ if changed is not None and pc . any ( changed ). as_py () :
136+ return source_table .take (pc . filter ( source_indices , changed ) )
123137 else :
124138 return source_table .schema .empty_table ()
0 commit comments