From cdf723378ee4b261433fcc67d46a6c34cb73c1f5 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Sat, 21 Nov 2026 01:44:27 +0530 Subject: [PATCH] Core: Allow limiting upsert update detection to selected columns Upsert compared every non-key column to decide whether a matched row changed, which is expensive for wide or nested rows. Add an optional difference_cols argument to get_rows_to_update and both upsert entry points so callers can compare only the columns that signal a change, such as a hash column. The argument is appended to the signatures to keep positional calls working. Unknown columns and key-only selections raise a ValueError. --- pyiceberg/table/__init__.py | 11 ++++++- pyiceberg/table/upsert_util.py | 20 ++++++++++-- tests/table/test_upsert.py | 59 ++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..b2231cf075 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -892,12 +892,16 @@ def upsert( case_sensitive: bool = True, branch: str | None = MAIN_BRANCH, snapshot_properties: dict[str, str] = EMPTY_DICT, + difference_cols: list[str] | None = None, ) -> UpsertResult: """Shorthand API for performing an upsert to an iceberg table. Args: df: The input dataframe to upsert with the table's data. join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. + difference_cols: Columns to compare when deciding whether a matched row changed. Defaults to + every non-key column. Restricting the comparison helps when only some columns, such as a + hash column, signal a change. when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any @@ -994,7 +998,7 @@ def upsert( # values have actually changed. We don't want to do just a blanket overwrite for matched # rows if the actual non-key column data hasn't changed. # this extra step avoids unnecessary IO and writes - rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols) + rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols, difference_cols) if len(rows_to_update) > 0: # build the match predicate filter @@ -1769,12 +1773,16 @@ def upsert( case_sensitive: bool = True, branch: str | None = MAIN_BRANCH, snapshot_properties: dict[str, str] = EMPTY_DICT, + difference_cols: list[str] | None = None, ) -> UpsertResult: """Shorthand API for performing an upsert to an iceberg table. Args: df: The input dataframe to upsert with the table's data. join_cols: Columns to join on, if not provided, it will use the identifier-field-ids. + difference_cols: Columns to compare when deciding whether a matched row changed. Defaults to + every non-key column. Restricting the comparison helps when only some columns, such as a + hash column, signal a change. when_matched_update_all: Bool indicating to update rows that are matched but require an update due to a value in a non-key column changing when_not_matched_insert_all: Bool indicating new rows to be inserted that do not match any @@ -1810,6 +1818,7 @@ def upsert( return tx.upsert( df=df, join_cols=join_cols, + difference_cols=difference_cols, when_matched_update_all=when_matched_update_all, when_not_matched_insert_all=when_not_matched_insert_all, case_sensitive=case_sensitive, diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 1a5692fa69..16b912f102 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -72,17 +72,33 @@ def _values_differ( return pc.invert(pc.or_(equal, both_null)) -def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table: +def get_rows_to_update( + source_table: pa.Table, target_table: pa.Table, join_cols: list[str], difference_cols: list[str] | None = None +) -> pa.Table: """ Return a table with rows that need to be updated in the target table based on the join columns. The table is joined on the identifier columns, and then checked if there are any updated rows. Those are selected and everything is renamed correctly. + + When ``difference_cols`` is set, only those columns are compared. This avoids comparing every + non-key column when the caller already knows which columns signal a change, such as a hash column. """ all_columns = set(source_table.column_names) join_cols_set = set(join_cols) non_key_cols = list(all_columns - join_cols_set) + compare_cols = non_key_cols + if difference_cols is not None: + difference_cols_set = set(difference_cols) + unknown_cols = difference_cols_set - all_columns + if unknown_cols: + raise ValueError(f"Difference columns not found in the source table: {sorted(unknown_cols)}") + compare_cols = [ + column for column in source_table.column_names if column in difference_cols_set and column not in join_cols_set + ] + if not compare_cols: + raise ValueError("Difference columns must include at least one non-key column") if has_duplicate_rows(target_table, join_cols): raise ValueError("Target table has duplicate rows, aborting upsert") @@ -124,7 +140,7 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols target_indices = matching_indices[TARGET_INDEX_COLUMN_NAME] changed: pa.Array | pa.ChunkedArray | None = None - for key in non_key_cols: + for key in compare_cols: column_changed = _values_differ( source_table.column(key).take(source_indices), target_table.column(key).take(target_indices), diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..a128e691ac 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -391,6 +391,33 @@ def test_upsert_with_identifier_fields(catalog: Catalog) -> None: assert [snap.summary.operation for snap in tbl.snapshots() if snap.summary is not None] == expected_operations +def test_upsert_with_difference_cols(catalog: Catalog) -> None: + identifier = "default.test_upsert_with_difference_cols" + _drop_table(catalog, identifier) + + schema = Schema( + NestedField(1, "city", StringType(), required=True), + NestedField(2, "population", IntegerType(), required=True), + NestedField(3, "note", StringType(), required=False), + identifier_field_ids=[1], + ) + arrow_schema = schema_to_pyarrow(schema) + tbl = catalog.create_table(identifier, schema=schema) + tbl.append(pa.Table.from_pylist([{"city": "Paris", "population": 1, "note": "a"}], schema=arrow_schema)) + + # population changed, which is the only compared column + df = pa.Table.from_pylist([{"city": "Paris", "population": 2, "note": "a"}], schema=arrow_schema) + result = tbl.upsert(df, difference_cols=["population"]) + assert result.rows_updated == 1 + + # note changed, but it is not compared + df = pa.Table.from_pylist([{"city": "Paris", "population": 2, "note": "b"}], schema=arrow_schema) + result = tbl.upsert(df, difference_cols=["population"]) + assert result.rows_updated == 0 + + assert tbl.scan().to_arrow().to_pylist() == [{"city": "Paris", "population": 2, "note": "a"}] + + def test_upsert_into_empty_table(catalog: Catalog) -> None: identifier = "default.test_upsert_into_empty_table" _drop_table(catalog, identifier) @@ -955,3 +982,35 @@ def test_get_rows_to_update_compares_nested_values() -> None: target = pa.table({"pk": [1, 2], "value": pa.array([{"a": 1}, {"a": 3}])}) assert get_rows_to_update(source, target, ["pk"]).to_pydict() == {"pk": [2], "value": [{"a": 2}]} + + +def test_get_rows_to_update_only_compares_difference_cols() -> None: + source = pa.table({"pk": [1, 2], "value": [10, 21], "ignored": [1, 2]}) + target = pa.table({"pk": [1, 2], "value": [10, 20], "ignored": [9, 9]}) + + assert get_rows_to_update(source, target, ["pk"], difference_cols=["value"]).to_pydict() == { + "pk": [2], + "value": [21], + "ignored": [2], + } + + +def test_get_rows_to_update_returns_nothing_when_difference_cols_are_unchanged() -> None: + source = pa.table({"pk": [1], "value": [10], "ignored": [1]}) + target = pa.table({"pk": [1], "value": [10], "ignored": [2]}) + + assert len(get_rows_to_update(source, target, ["pk"], difference_cols=["value"])) == 0 + + +def test_get_rows_to_update_rejects_unknown_difference_cols() -> None: + source = pa.table({"pk": [1], "value": [10]}) + + with pytest.raises(ValueError, match="Difference columns not found"): + get_rows_to_update(source, source, ["pk"], difference_cols=["missing"]) + + +def test_get_rows_to_update_rejects_key_only_difference_cols() -> None: + source = pa.table({"pk": [1], "value": [10]}) + + with pytest.raises(ValueError, match="at least one non-key column"): + get_rows_to_update(source, source, ["pk"], difference_cols=["pk"])