Skip to content

Commit 32beb5d

Browse files
fix: поддержать upsert после добавления столбца
Co-authored-by: gpt-6.2-sol <noreply@openai.com>
1 parent 07216ed commit 32beb5d

4 files changed

Lines changed: 185 additions & 25 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3141,34 +3141,38 @@ def _check_pyarrow_schema_compatible(
31413141
provided_schema: pa.Schema,
31423142
downcast_ns_timestamp_to_us: bool = False,
31433143
format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION,
3144-
) -> None:
3144+
) -> Schema:
31453145
"""
31463146
Check if the `requested_schema` is compatible with `provided_schema`.
31473147
31483148
Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type.
31493149
3150+
Returns:
3151+
The provided schema with Iceberg field IDs.
3152+
31503153
Raises:
31513154
ValueError: If the schemas are not compatible.
31523155
"""
31533156
name_mapping = requested_schema.name_mapping
31543157

31553158
try:
3156-
provided_schema = pyarrow_to_schema(
3159+
provided_iceberg_schema = pyarrow_to_schema(
31573160
provided_schema,
31583161
name_mapping=name_mapping,
31593162
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
31603163
format_version=format_version,
31613164
)
31623165
except ValueError as e:
3163-
provided_schema = _pyarrow_to_schema_without_ids(
3166+
provided_iceberg_schema = _pyarrow_to_schema_without_ids(
31643167
provided_schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version
31653168
)
3166-
additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys())
3169+
additional_names = set(provided_iceberg_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys())
31673170
raise ValueError(
31683171
f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. "
31693172
"Update the schema first (hint, use union_by_name)."
31703173
) from e
3171-
_check_schema_compatible(requested_schema, provided_schema)
3174+
_check_schema_compatible(requested_schema, provided_iceberg_schema)
3175+
return provided_iceberg_schema
31723176

31733177

31743178
def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]:

‎pyiceberg/table/__init__.py‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@
5757
ManifestFile,
5858
)
5959
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, UNPARTITIONED_PARTITION_SPEC, PartitionKey, PartitionSpec
60-
from pyiceberg.schema import Schema
60+
from pyiceberg.schema import Schema, prune_columns
6161
from pyiceberg.table.delete_file_index import DeleteFileIndex
6262
from pyiceberg.table.inspect import InspectTable
6363
from pyiceberg.table.labels import Labels
@@ -955,11 +955,12 @@ def upsert(
955955
if upsert_util.has_duplicate_rows(df, join_cols):
956956
raise ValueError("Duplicate rows found in source dataset based on the key columns. No upsert executed")
957957

958-
from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible
958+
from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _to_requested_schema
959959

960960
downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False
961-
_check_pyarrow_schema_compatible(
962-
self.table_metadata.schema(),
961+
table_schema = self.table_metadata.schema()
962+
source_schema = _check_pyarrow_schema_compatible(
963+
table_schema,
963964
provided_schema=df.schema,
964965
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
965966
format_version=self.table_metadata.format_version,
@@ -980,7 +981,41 @@ def upsert(
980981
if branch in self.table_metadata.refs:
981982
matched_iceberg_record_batches_scan = matched_iceberg_record_batches_scan.use_ref(branch)
982983

983-
matched_iceberg_record_batches = matched_iceberg_record_batches_scan.to_arrow_batch_reader()
984+
# Compare only the supplied fields, using current field types even for older snapshots.
985+
comparison_schema = table_schema
986+
if source_schema.field_ids != table_schema.field_ids:
987+
supplied_fields = {
988+
field_id
989+
for field_id in source_schema.field_ids
990+
if (field_type := source_schema.find_type(field_id)).is_primitive or field_type.is_struct
991+
}
992+
comparison_schema = prune_columns(table_schema, supplied_fields, select_full_types=False)
993+
994+
# Align renamed fields and omit dropped fields for comparison while keeping the original rows for writes.
995+
comparison_source = None
996+
if (
997+
when_matched_update_all
998+
and df.num_rows
999+
and any(
1000+
source_schema.find_column_name(field_id) != comparison_schema.find_column_name(field_id)
1001+
for field_id in source_schema.field_ids
1002+
)
1003+
):
1004+
comparison_source = pa.Table.from_batches(
1005+
_to_requested_schema(
1006+
comparison_schema,
1007+
source_schema,
1008+
batch,
1009+
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
1010+
)
1011+
for batch in df.to_batches()
1012+
)
1013+
1014+
matched_iceberg_record_batches = _to_arrow_batch_reader_via_file_scan_tasks(
1015+
matched_iceberg_record_batches_scan,
1016+
comparison_schema,
1017+
matched_iceberg_record_batches_scan.plan_files(),
1018+
)
9841019

9851020
batches_to_overwrite = []
9861021
overwrite_predicates = []
@@ -994,7 +1029,7 @@ def upsert(
9941029
# values have actually changed. We don't want to do just a blanket overwrite for matched
9951030
# rows if the actual non-key column data hasn't changed.
9961031
# this extra step avoids unnecessary IO and writes
997-
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols)
1032+
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols, comparison_source=comparison_source)
9981033

9991034
if len(rows_to_update) > 0:
10001035
# build the match predicate filter
@@ -1005,7 +1040,7 @@ def upsert(
10051040

10061041
if when_not_matched_insert_all:
10071042
expr_match = upsert_util.create_match_filter(rows, join_cols)
1008-
expr_match_bound = bind(self.table_metadata.schema(), expr_match, case_sensitive=case_sensitive)
1043+
expr_match_bound = bind(table_schema, expr_match, case_sensitive=case_sensitive)
10091044
expr_match_arrow = expression_to_pyarrow(expr_match_bound)
10101045

10111046
# Filter rows per batch.

‎pyiceberg/table/upsert_util.py‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -72,17 +72,18 @@ def _values_differ(
7272
return pc.invert(pc.or_(equal, both_null))
7373

7474

75-
def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table:
75+
def get_rows_to_update(
76+
source_table: pa.Table, target_table: pa.Table, join_cols: list[str], *, comparison_source: pa.Table | None = None
77+
) -> pa.Table:
7678
"""
7779
Return a table with rows that need to be updated in the target table based on the join columns.
7880
7981
The table is joined on the identifier columns, and then checked if there are any updated rows.
80-
Those are selected and everything is renamed correctly.
82+
If supplied, `comparison_source` contains the same rows in the same order, with field names aligned to the target.
83+
The selected rows come from the original source table.
8184
"""
82-
all_columns = set(source_table.column_names)
83-
join_cols_set = set(join_cols)
84-
85-
non_key_cols = list(all_columns - join_cols_set)
85+
values_to_compare = source_table if comparison_source is None else comparison_source
86+
non_key_cols = set(values_to_compare.column_names) - set(join_cols)
8687

8788
if has_duplicate_rows(target_table, join_cols):
8889
raise ValueError("Target table has duplicate rows, aborting upsert")
@@ -105,19 +106,20 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
105106
) from None
106107

107108
# Step 1: Prepare source index with join keys and a marker index
108-
# Cast to target table schema, so we can do the join
109+
# Cast only the join keys because the source can omit optional target columns.
109110
# See: https://github.com/apache/arrow/issues/37542
111+
target_keys = target_table.select(join_cols)
110112
source_index = (
111-
source_table.cast(target_table.schema)
112-
.select(join_cols_set)
113+
source_table.select(join_cols)
114+
.cast(target_keys.schema)
113115
.append_column(SOURCE_INDEX_COLUMN_NAME, pa.array(range(len(source_table))))
114116
)
115117

116118
# Step 2: Prepare target index with join keys and a marker
117-
target_index = target_table.select(join_cols_set).append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table))))
119+
target_index = target_keys.append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table))))
118120

119121
# Step 3: Perform an inner join to find which rows from source exist in target
120-
matching_indices = source_index.join(target_index, keys=list(join_cols_set), join_type="inner")
122+
matching_indices = source_index.join(target_index, keys=join_cols, join_type="inner")
121123

122124
# Step 4: Compare the matched rows column by column
123125
source_indices = matching_indices[SOURCE_INDEX_COLUMN_NAME]
@@ -126,12 +128,12 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
126128
changed: pa.Array | pa.ChunkedArray | None = None
127129
for key in non_key_cols:
128130
column_changed = _values_differ(
129-
source_table.column(key).take(source_indices),
131+
values_to_compare.column(key).take(source_indices),
130132
target_table.column(key).take(target_indices),
131133
)
132134
changed = column_changed if changed is None else pc.or_(changed, column_changed)
133135

134-
# Step 5: Take rows from source table using the indices and cast to target schema
136+
# Step 5: Take rows from source table using the indices
135137
if changed is not None and pc.any(changed).as_py():
136138
return source_table.take(pc.filter(source_indices, changed))
137139
else:

‎tests/table/test_upsert.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -674,6 +674,125 @@ def test_upsert_with_struct_field_as_join_key(catalog: Catalog) -> None:
674674
_ = tbl.upsert(update_data, join_cols=["nested_type"])
675675

676676

677+
@pytest.mark.parametrize("branch", ["main", "dev"])
678+
def test_upsert_after_union_by_name(catalog: Catalog, branch: str) -> None:
679+
original = pa.table({"id": [1, 2], "value": ["one", "two"]})
680+
table = catalog.create_table("default.test_upsert_after_union_by_name", original.schema)
681+
table.append(original)
682+
snapshot = table.current_snapshot()
683+
assert snapshot is not None
684+
table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit()
685+
686+
source = pa.table({"id": [1, 2, 3], "value": ["one", "two", "three"], "note": ["updated", None, "inserted"]})
687+
with table.update_schema() as update_schema:
688+
update_schema.union_by_name(source.schema)
689+
690+
result = table.upsert(source, join_cols=["id"], branch=branch)
691+
692+
assert_upsert_result(result, expected_updated=1, expected_inserted=1)
693+
assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
694+
{"id": 1, "value": "one", "note": "updated"},
695+
{"id": 2, "value": "two", "note": None},
696+
{"id": 3, "value": "three", "note": "inserted"},
697+
]
698+
699+
700+
@pytest.mark.parametrize("branch", ["main", "dev"])
701+
def test_upsert_without_new_optional_column(catalog: Catalog, branch: str) -> None:
702+
original = pa.table({"id": [1, 2], "value": ["one", "two"]})
703+
table = catalog.create_table("default.test_upsert_without_new_optional_column", original.schema)
704+
table.append(original)
705+
snapshot = table.current_snapshot()
706+
assert snapshot is not None
707+
table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit()
708+
table.update_schema().add_column("note", StringType()).commit()
709+
710+
source = pa.table({"id": pa.array([1, 2, 3], type=pa.int32()), "value": ["updated", "two", "inserted"]})
711+
result = table.upsert(source, join_cols=["id"], branch=branch)
712+
713+
assert_upsert_result(result, expected_updated=1, expected_inserted=1)
714+
assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
715+
{"id": 1, "value": "updated", "note": None},
716+
{"id": 2, "value": "two", "note": None},
717+
{"id": 3, "value": "inserted", "note": None},
718+
]
719+
720+
721+
def test_upsert_without_new_optional_struct_field(catalog: Catalog) -> None:
722+
original = pa.table({"id": [1, 2], "payload": [{"value": "one"}, {"value": "two"}]})
723+
table = catalog.create_table("default.test_upsert_without_new_optional_struct_field", original.schema)
724+
table.append(original)
725+
table.update_schema().add_column(("payload", "note"), StringType()).commit()
726+
727+
source = pa.table({"id": [1, 2, 3], "payload": [{"value": "updated"}, {"value": "two"}, {"value": "inserted"}]})
728+
result = table.upsert(source, join_cols=["id"])
729+
730+
assert_upsert_result(result, expected_updated=1, expected_inserted=1)
731+
assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
732+
{"id": 1, "payload": {"value": "updated", "note": None}},
733+
{"id": 2, "payload": {"value": "two", "note": None}},
734+
{"id": 3, "payload": {"value": "inserted", "note": None}},
735+
]
736+
737+
738+
def test_upsert_matches_renamed_struct_fields_by_id(catalog: Catalog) -> None:
739+
original = pa.table({"id": [1], "payload": [{"a": 1, "b": 2}]})
740+
table = catalog.create_table("default.test_upsert_matches_renamed_struct_fields_by_id", original.schema)
741+
table.append(original)
742+
original_schema = table.schema().as_arrow()
743+
with table.update_schema() as update_schema:
744+
update_schema.rename_column(("payload", "a"), "b")
745+
update_schema.rename_column(("payload", "b"), "a")
746+
747+
source = pa.Table.from_pylist([{"id": 1, "payload": {"a": 2, "b": 1}}], schema=original_schema)
748+
result = table.upsert(source, join_cols=["id"])
749+
750+
assert_upsert_result(result, expected_updated=1, expected_inserted=0)
751+
assert table.scan().to_arrow().to_pylist() == [{"id": 1, "payload": {"b": 2, "a": 1}}]
752+
assert_upsert_result(table.upsert(source, join_cols=["id"]), expected_updated=0, expected_inserted=0)
753+
754+
755+
def test_upsert_matches_renamed_column_by_id(catalog: Catalog) -> None:
756+
original = pa.table({"id": [1], "value": ["old"]})
757+
table = catalog.create_table("default.test_upsert_matches_renamed_column_by_id", original.schema)
758+
table.append(original)
759+
original_schema = table.schema().as_arrow()
760+
table.update_schema().rename_column("value", "renamed").commit()
761+
762+
source = pa.Table.from_pylist([{"id": 1, "value": "updated"}], schema=original_schema)
763+
result = table.upsert(source, join_cols=["id"])
764+
765+
assert_upsert_result(result, expected_updated=1, expected_inserted=0)
766+
assert table.scan().to_arrow().to_pylist() == [{"id": 1, "renamed": "updated"}]
767+
768+
769+
def test_upsert_ignores_dropped_field_with_reused_name(catalog: Catalog) -> None:
770+
original = pa.table({"id": [1, 2], "value": ["old one", "old two"], "label": ["one", "two"]})
771+
table = catalog.create_table("default.test_upsert_ignores_dropped_field_with_reused_name", original.schema)
772+
table.append(original)
773+
original_schema = table.schema()
774+
table.update_schema().delete_column("value").commit()
775+
table.update_schema().add_column("value", StringType()).commit()
776+
assert table.schema().find_field("value").field_id != original_schema.find_field("value").field_id
777+
778+
source = pa.Table.from_pylist(
779+
[
780+
{"id": 1, "value": "stale update", "label": "updated"},
781+
{"id": 2, "value": "stale unchanged", "label": "two"},
782+
{"id": 3, "value": "stale insert", "label": "inserted"},
783+
],
784+
schema=original_schema.as_arrow(),
785+
)
786+
result = table.upsert(source, join_cols=["id"])
787+
788+
assert_upsert_result(result, expected_updated=1, expected_inserted=1)
789+
assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
790+
{"id": 1, "label": "updated", "value": None},
791+
{"id": 2, "label": "two", "value": None},
792+
{"id": 3, "label": "inserted", "value": None},
793+
]
794+
795+
677796
def test_upsert_with_nulls(catalog: Catalog) -> None:
678797
identifier = "default.test_upsert_with_nulls"
679798
_drop_table(catalog, identifier)

0 commit comments

Comments
 (0)