diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..4328593b7f 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3141,34 +3141,38 @@ def _check_pyarrow_schema_compatible( provided_schema: pa.Schema, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, -) -> None: +) -> Schema: """ Check if the `requested_schema` is compatible with `provided_schema`. Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type. + Returns: + The provided schema with Iceberg field IDs. + Raises: ValueError: If the schemas are not compatible. """ name_mapping = requested_schema.name_mapping try: - provided_schema = pyarrow_to_schema( + provided_iceberg_schema = pyarrow_to_schema( provided_schema, name_mapping=name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version, ) except ValueError as e: - provided_schema = _pyarrow_to_schema_without_ids( + provided_iceberg_schema = _pyarrow_to_schema_without_ids( provided_schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version ) - additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) + additional_names = set(provided_iceberg_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys()) raise ValueError( f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. " "Update the schema first (hint, use union_by_name)." ) from e - _check_schema_compatible(requested_schema, provided_schema) + _check_schema_compatible(requested_schema, provided_iceberg_schema) + return provided_iceberg_schema def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..638a811c6f 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -57,7 +57,7 @@ ManifestFile, ) from pyiceberg.partitioning import PARTITION_FIELD_ID_START, UNPARTITIONED_PARTITION_SPEC, PartitionKey, PartitionSpec -from pyiceberg.schema import Schema +from pyiceberg.schema import Schema, prune_columns from pyiceberg.table.delete_file_index import DeleteFileIndex from pyiceberg.table.inspect import InspectTable from pyiceberg.table.labels import Labels @@ -955,11 +955,12 @@ def upsert( if upsert_util.has_duplicate_rows(df, join_cols): raise ValueError("Duplicate rows found in source dataset based on the key columns. No upsert executed") - from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible + from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _to_requested_schema downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False - _check_pyarrow_schema_compatible( - self.table_metadata.schema(), + table_schema = self.table_metadata.schema() + source_schema = _check_pyarrow_schema_compatible( + table_schema, provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, @@ -980,7 +981,41 @@ def upsert( if branch in self.table_metadata.refs: matched_iceberg_record_batches_scan = matched_iceberg_record_batches_scan.use_ref(branch) - matched_iceberg_record_batches = matched_iceberg_record_batches_scan.to_arrow_batch_reader() + # Compare only the supplied fields, using current field types even for older snapshots. + comparison_schema = table_schema + if source_schema.field_ids != table_schema.field_ids: + supplied_fields = { + field_id + for field_id in source_schema.field_ids + if (field_type := source_schema.find_type(field_id)).is_primitive or field_type.is_struct + } + comparison_schema = prune_columns(table_schema, supplied_fields, select_full_types=False) + + # Align renamed fields and omit dropped fields for comparison while keeping the original rows for writes. + comparison_source = None + if ( + when_matched_update_all + and df.num_rows + and any( + source_schema.find_column_name(field_id) != comparison_schema.find_column_name(field_id) + for field_id in source_schema.field_ids + ) + ): + comparison_source = pa.Table.from_batches( + _to_requested_schema( + comparison_schema, + source_schema, + batch, + downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, + ) + for batch in df.to_batches() + ) + + matched_iceberg_record_batches = _to_arrow_batch_reader_via_file_scan_tasks( + matched_iceberg_record_batches_scan, + comparison_schema, + matched_iceberg_record_batches_scan.plan_files(), + ) batches_to_overwrite = [] overwrite_predicates = [] @@ -994,7 +1029,7 @@ def upsert( # values have actually changed. We don't want to do just a blanket overwrite for matched # rows if the actual non-key column data hasn't changed. # this extra step avoids unnecessary IO and writes - rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols) + rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols, comparison_source=comparison_source) if len(rows_to_update) > 0: # build the match predicate filter @@ -1005,7 +1040,7 @@ def upsert( if when_not_matched_insert_all: expr_match = upsert_util.create_match_filter(rows, join_cols) - expr_match_bound = bind(self.table_metadata.schema(), expr_match, case_sensitive=case_sensitive) + expr_match_bound = bind(table_schema, expr_match, case_sensitive=case_sensitive) expr_match_arrow = expression_to_pyarrow(expr_match_bound) # Filter rows per batch. diff --git a/pyiceberg/table/upsert_util.py b/pyiceberg/table/upsert_util.py index 1a5692fa69..6c62b28bf9 100644 --- a/pyiceberg/table/upsert_util.py +++ b/pyiceberg/table/upsert_util.py @@ -72,17 +72,18 @@ def _values_differ( return pc.invert(pc.or_(equal, both_null)) -def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table: +def get_rows_to_update( + source_table: pa.Table, target_table: pa.Table, join_cols: list[str], *, comparison_source: pa.Table | None = None +) -> pa.Table: """ Return a table with rows that need to be updated in the target table based on the join columns. The table is joined on the identifier columns, and then checked if there are any updated rows. - Those are selected and everything is renamed correctly. + If supplied, `comparison_source` contains the same rows in the same order, with field names aligned to the target. + The selected rows come from the original source table. """ - all_columns = set(source_table.column_names) - join_cols_set = set(join_cols) - - non_key_cols = list(all_columns - join_cols_set) + values_to_compare = source_table if comparison_source is None else comparison_source + non_key_cols = set(values_to_compare.column_names) - set(join_cols) if has_duplicate_rows(target_table, join_cols): raise ValueError("Target table has duplicate rows, aborting upsert") @@ -105,19 +106,20 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols ) from None # Step 1: Prepare source index with join keys and a marker index - # Cast to target table schema, so we can do the join + # Cast only the join keys because the source can omit optional target columns. # See: https://github.com/apache/arrow/issues/37542 + target_keys = target_table.select(join_cols) source_index = ( - source_table.cast(target_table.schema) - .select(join_cols_set) + source_table.select(join_cols) + .cast(target_keys.schema) .append_column(SOURCE_INDEX_COLUMN_NAME, pa.array(range(len(source_table)))) ) # Step 2: Prepare target index with join keys and a marker - target_index = target_table.select(join_cols_set).append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table)))) + target_index = target_keys.append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table)))) # Step 3: Perform an inner join to find which rows from source exist in target - matching_indices = source_index.join(target_index, keys=list(join_cols_set), join_type="inner") + matching_indices = source_index.join(target_index, keys=join_cols, join_type="inner") # Step 4: Compare the matched rows column by column source_indices = matching_indices[SOURCE_INDEX_COLUMN_NAME] @@ -126,12 +128,12 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols changed: pa.Array | pa.ChunkedArray | None = None for key in non_key_cols: column_changed = _values_differ( - source_table.column(key).take(source_indices), + values_to_compare.column(key).take(source_indices), target_table.column(key).take(target_indices), ) changed = column_changed if changed is None else pc.or_(changed, column_changed) - # Step 5: Take rows from source table using the indices and cast to target schema + # Step 5: Take rows from source table using the indices if changed is not None and pc.any(changed).as_py(): return source_table.take(pc.filter(source_indices, changed)) else: diff --git a/tests/table/test_upsert.py b/tests/table/test_upsert.py index 4291c18a55..76a5a1ca9b 100644 --- a/tests/table/test_upsert.py +++ b/tests/table/test_upsert.py @@ -674,6 +674,125 @@ def test_upsert_with_struct_field_as_join_key(catalog: Catalog) -> None: _ = tbl.upsert(update_data, join_cols=["nested_type"]) +@pytest.mark.parametrize("branch", ["main", "dev"]) +def test_upsert_after_union_by_name(catalog: Catalog, branch: str) -> None: + original = pa.table({"id": [1, 2], "value": ["one", "two"]}) + table = catalog.create_table("default.test_upsert_after_union_by_name", original.schema) + table.append(original) + snapshot = table.current_snapshot() + assert snapshot is not None + table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit() + + source = pa.table({"id": [1, 2, 3], "value": ["one", "two", "three"], "note": ["updated", None, "inserted"]}) + with table.update_schema() as update_schema: + update_schema.union_by_name(source.schema) + + result = table.upsert(source, join_cols=["id"], branch=branch) + + assert_upsert_result(result, expected_updated=1, expected_inserted=1) + assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [ + {"id": 1, "value": "one", "note": "updated"}, + {"id": 2, "value": "two", "note": None}, + {"id": 3, "value": "three", "note": "inserted"}, + ] + + +@pytest.mark.parametrize("branch", ["main", "dev"]) +def test_upsert_without_new_optional_column(catalog: Catalog, branch: str) -> None: + original = pa.table({"id": [1, 2], "value": ["one", "two"]}) + table = catalog.create_table("default.test_upsert_without_new_optional_column", original.schema) + table.append(original) + snapshot = table.current_snapshot() + assert snapshot is not None + table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit() + table.update_schema().add_column("note", StringType()).commit() + + source = pa.table({"id": pa.array([1, 2, 3], type=pa.int32()), "value": ["updated", "two", "inserted"]}) + result = table.upsert(source, join_cols=["id"], branch=branch) + + assert_upsert_result(result, expected_updated=1, expected_inserted=1) + assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [ + {"id": 1, "value": "updated", "note": None}, + {"id": 2, "value": "two", "note": None}, + {"id": 3, "value": "inserted", "note": None}, + ] + + +def test_upsert_without_new_optional_struct_field(catalog: Catalog) -> None: + original = pa.table({"id": [1, 2], "payload": [{"value": "one"}, {"value": "two"}]}) + table = catalog.create_table("default.test_upsert_without_new_optional_struct_field", original.schema) + table.append(original) + table.update_schema().add_column(("payload", "note"), StringType()).commit() + + source = pa.table({"id": [1, 2, 3], "payload": [{"value": "updated"}, {"value": "two"}, {"value": "inserted"}]}) + result = table.upsert(source, join_cols=["id"]) + + assert_upsert_result(result, expected_updated=1, expected_inserted=1) + assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [ + {"id": 1, "payload": {"value": "updated", "note": None}}, + {"id": 2, "payload": {"value": "two", "note": None}}, + {"id": 3, "payload": {"value": "inserted", "note": None}}, + ] + + +def test_upsert_matches_renamed_struct_fields_by_id(catalog: Catalog) -> None: + original = pa.table({"id": [1], "payload": [{"a": 1, "b": 2}]}) + table = catalog.create_table("default.test_upsert_matches_renamed_struct_fields_by_id", original.schema) + table.append(original) + original_schema = table.schema().as_arrow() + with table.update_schema() as update_schema: + update_schema.rename_column(("payload", "a"), "b") + update_schema.rename_column(("payload", "b"), "a") + + source = pa.Table.from_pylist([{"id": 1, "payload": {"a": 2, "b": 1}}], schema=original_schema) + result = table.upsert(source, join_cols=["id"]) + + assert_upsert_result(result, expected_updated=1, expected_inserted=0) + assert table.scan().to_arrow().to_pylist() == [{"id": 1, "payload": {"b": 2, "a": 1}}] + assert_upsert_result(table.upsert(source, join_cols=["id"]), expected_updated=0, expected_inserted=0) + + +def test_upsert_matches_renamed_column_by_id(catalog: Catalog) -> None: + original = pa.table({"id": [1], "value": ["old"]}) + table = catalog.create_table("default.test_upsert_matches_renamed_column_by_id", original.schema) + table.append(original) + original_schema = table.schema().as_arrow() + table.update_schema().rename_column("value", "renamed").commit() + + source = pa.Table.from_pylist([{"id": 1, "value": "updated"}], schema=original_schema) + result = table.upsert(source, join_cols=["id"]) + + assert_upsert_result(result, expected_updated=1, expected_inserted=0) + assert table.scan().to_arrow().to_pylist() == [{"id": 1, "renamed": "updated"}] + + +def test_upsert_ignores_dropped_field_with_reused_name(catalog: Catalog) -> None: + original = pa.table({"id": [1, 2], "value": ["old one", "old two"], "label": ["one", "two"]}) + table = catalog.create_table("default.test_upsert_ignores_dropped_field_with_reused_name", original.schema) + table.append(original) + original_schema = table.schema() + table.update_schema().delete_column("value").commit() + table.update_schema().add_column("value", StringType()).commit() + assert table.schema().find_field("value").field_id != original_schema.find_field("value").field_id + + source = pa.Table.from_pylist( + [ + {"id": 1, "value": "stale update", "label": "updated"}, + {"id": 2, "value": "stale unchanged", "label": "two"}, + {"id": 3, "value": "stale insert", "label": "inserted"}, + ], + schema=original_schema.as_arrow(), + ) + result = table.upsert(source, join_cols=["id"]) + + assert_upsert_result(result, expected_updated=1, expected_inserted=1) + assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [ + {"id": 1, "label": "updated", "value": None}, + {"id": 2, "label": "two", "value": None}, + {"id": 3, "label": "inserted", "value": None}, + ] + + def test_upsert_with_nulls(catalog: Catalog) -> None: identifier = "default.test_upsert_with_nulls" _drop_table(catalog, identifier)