Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3141,34 +3141,38 @@ def _check_pyarrow_schema_compatible(
provided_schema: pa.Schema,
downcast_ns_timestamp_to_us: bool = False,
format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION,
) -> None:
) -> Schema:
"""
Check if the `requested_schema` is compatible with `provided_schema`.

Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type.

Returns:
The provided schema with Iceberg field IDs.

Raises:
ValueError: If the schemas are not compatible.
"""
name_mapping = requested_schema.name_mapping

try:
provided_schema = pyarrow_to_schema(
provided_iceberg_schema = pyarrow_to_schema(
provided_schema,
name_mapping=name_mapping,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=format_version,
)
except ValueError as e:
provided_schema = _pyarrow_to_schema_without_ids(
provided_iceberg_schema = _pyarrow_to_schema_without_ids(
provided_schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version
)
additional_names = set(provided_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys())
additional_names = set(provided_iceberg_schema._name_to_id.keys()) - set(requested_schema._name_to_id.keys())
raise ValueError(
f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. "
"Update the schema first (hint, use union_by_name)."
) from e
_check_schema_compatible(requested_schema, provided_schema)
_check_schema_compatible(requested_schema, provided_iceberg_schema)
return provided_iceberg_schema


def parquet_files_to_data_files(io: FileIO, table_metadata: TableMetadata, file_paths: Iterator[str]) -> Iterator[DataFile]:
Expand Down
49 changes: 42 additions & 7 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
ManifestFile,
)
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, UNPARTITIONED_PARTITION_SPEC, PartitionKey, PartitionSpec
from pyiceberg.schema import Schema
from pyiceberg.schema import Schema, prune_columns
from pyiceberg.table.delete_file_index import DeleteFileIndex
from pyiceberg.table.inspect import InspectTable
from pyiceberg.table.labels import Labels
Expand Down Expand Up @@ -955,11 +955,12 @@ def upsert(
if upsert_util.has_duplicate_rows(df, join_cols):
raise ValueError("Duplicate rows found in source dataset based on the key columns. No upsert executed")

from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible
from pyiceberg.io.pyarrow import _check_pyarrow_schema_compatible, _to_requested_schema

downcast_ns_timestamp_to_us = Config().get_bool(DOWNCAST_NS_TIMESTAMP_TO_US_ON_WRITE) or False
_check_pyarrow_schema_compatible(
self.table_metadata.schema(),
table_schema = self.table_metadata.schema()
source_schema = _check_pyarrow_schema_compatible(
table_schema,
provided_schema=df.schema,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=self.table_metadata.format_version,
Expand All @@ -980,7 +981,41 @@ def upsert(
if branch in self.table_metadata.refs:
matched_iceberg_record_batches_scan = matched_iceberg_record_batches_scan.use_ref(branch)

matched_iceberg_record_batches = matched_iceberg_record_batches_scan.to_arrow_batch_reader()
# Compare only the supplied fields, using current field types even for older snapshots.
comparison_schema = table_schema
if source_schema.field_ids != table_schema.field_ids:
supplied_fields = {
field_id
for field_id in source_schema.field_ids
if (field_type := source_schema.find_type(field_id)).is_primitive or field_type.is_struct
}
comparison_schema = prune_columns(table_schema, supplied_fields, select_full_types=False)

# Align renamed fields and omit dropped fields for comparison while keeping the original rows for writes.
comparison_source = None
if (
when_matched_update_all
and df.num_rows
and any(
source_schema.find_column_name(field_id) != comparison_schema.find_column_name(field_id)
for field_id in source_schema.field_ids
)
):
comparison_source = pa.Table.from_batches(
_to_requested_schema(
comparison_schema,
source_schema,
batch,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
)
for batch in df.to_batches()
)

matched_iceberg_record_batches = _to_arrow_batch_reader_via_file_scan_tasks(
matched_iceberg_record_batches_scan,
comparison_schema,
matched_iceberg_record_batches_scan.plan_files(),
)

batches_to_overwrite = []
overwrite_predicates = []
Expand All @@ -994,7 +1029,7 @@ def upsert(
# values have actually changed. We don't want to do just a blanket overwrite for matched
# rows if the actual non-key column data hasn't changed.
# this extra step avoids unnecessary IO and writes
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols)
rows_to_update = upsert_util.get_rows_to_update(df, rows, join_cols, comparison_source=comparison_source)

if len(rows_to_update) > 0:
# build the match predicate filter
Expand All @@ -1005,7 +1040,7 @@ def upsert(

if when_not_matched_insert_all:
expr_match = upsert_util.create_match_filter(rows, join_cols)
expr_match_bound = bind(self.table_metadata.schema(), expr_match, case_sensitive=case_sensitive)
expr_match_bound = bind(table_schema, expr_match, case_sensitive=case_sensitive)
expr_match_arrow = expression_to_pyarrow(expr_match_bound)

# Filter rows per batch.
Expand Down
28 changes: 15 additions & 13 deletions pyiceberg/table/upsert_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,18 @@ def _values_differ(
return pc.invert(pc.or_(equal, both_null))


def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols: list[str]) -> pa.Table:
def get_rows_to_update(
source_table: pa.Table, target_table: pa.Table, join_cols: list[str], *, comparison_source: pa.Table | None = None
) -> pa.Table:
"""
Return a table with rows that need to be updated in the target table based on the join columns.

The table is joined on the identifier columns, and then checked if there are any updated rows.
Those are selected and everything is renamed correctly.
If supplied, `comparison_source` contains the same rows in the same order, with field names aligned to the target.
The selected rows come from the original source table.
"""
all_columns = set(source_table.column_names)
join_cols_set = set(join_cols)

non_key_cols = list(all_columns - join_cols_set)
values_to_compare = source_table if comparison_source is None else comparison_source
non_key_cols = set(values_to_compare.column_names) - set(join_cols)

if has_duplicate_rows(target_table, join_cols):
raise ValueError("Target table has duplicate rows, aborting upsert")
Expand All @@ -105,19 +106,20 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
) from None

# Step 1: Prepare source index with join keys and a marker index
# Cast to target table schema, so we can do the join
# Cast only the join keys because the source can omit optional target columns.
# See: https://github.com/apache/arrow/issues/37542
target_keys = target_table.select(join_cols)
source_index = (
source_table.cast(target_table.schema)
.select(join_cols_set)
source_table.select(join_cols)
.cast(target_keys.schema)
.append_column(SOURCE_INDEX_COLUMN_NAME, pa.array(range(len(source_table))))
)

# Step 2: Prepare target index with join keys and a marker
target_index = target_table.select(join_cols_set).append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table))))
target_index = target_keys.append_column(TARGET_INDEX_COLUMN_NAME, pa.array(range(len(target_table))))

# Step 3: Perform an inner join to find which rows from source exist in target
matching_indices = source_index.join(target_index, keys=list(join_cols_set), join_type="inner")
matching_indices = source_index.join(target_index, keys=join_cols, join_type="inner")

# Step 4: Compare the matched rows column by column
source_indices = matching_indices[SOURCE_INDEX_COLUMN_NAME]
Expand All @@ -126,12 +128,12 @@ def get_rows_to_update(source_table: pa.Table, target_table: pa.Table, join_cols
changed: pa.Array | pa.ChunkedArray | None = None
for key in non_key_cols:
column_changed = _values_differ(
source_table.column(key).take(source_indices),
values_to_compare.column(key).take(source_indices),
target_table.column(key).take(target_indices),
)
changed = column_changed if changed is None else pc.or_(changed, column_changed)

# Step 5: Take rows from source table using the indices and cast to target schema
# Step 5: Take rows from source table using the indices
if changed is not None and pc.any(changed).as_py():
return source_table.take(pc.filter(source_indices, changed))
else:
Expand Down
119 changes: 119 additions & 0 deletions tests/table/test_upsert.py
Original file line number Diff line number Diff line change
Expand Up @@ -674,6 +674,125 @@ def test_upsert_with_struct_field_as_join_key(catalog: Catalog) -> None:
_ = tbl.upsert(update_data, join_cols=["nested_type"])


@pytest.mark.parametrize("branch", ["main", "dev"])
def test_upsert_after_union_by_name(catalog: Catalog, branch: str) -> None:
original = pa.table({"id": [1, 2], "value": ["one", "two"]})
table = catalog.create_table("default.test_upsert_after_union_by_name", original.schema)
table.append(original)
snapshot = table.current_snapshot()
assert snapshot is not None
table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit()

source = pa.table({"id": [1, 2, 3], "value": ["one", "two", "three"], "note": ["updated", None, "inserted"]})
with table.update_schema() as update_schema:
update_schema.union_by_name(source.schema)

result = table.upsert(source, join_cols=["id"], branch=branch)

assert_upsert_result(result, expected_updated=1, expected_inserted=1)
assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
{"id": 1, "value": "one", "note": "updated"},
{"id": 2, "value": "two", "note": None},
{"id": 3, "value": "three", "note": "inserted"},
]


@pytest.mark.parametrize("branch", ["main", "dev"])
def test_upsert_without_new_optional_column(catalog: Catalog, branch: str) -> None:
original = pa.table({"id": [1, 2], "value": ["one", "two"]})
table = catalog.create_table("default.test_upsert_without_new_optional_column", original.schema)
table.append(original)
snapshot = table.current_snapshot()
assert snapshot is not None
table.manage_snapshots().create_branch(snapshot_id=snapshot.snapshot_id, branch_name="dev").commit()
table.update_schema().add_column("note", StringType()).commit()

source = pa.table({"id": pa.array([1, 2, 3], type=pa.int32()), "value": ["updated", "two", "inserted"]})
result = table.upsert(source, join_cols=["id"], branch=branch)

assert_upsert_result(result, expected_updated=1, expected_inserted=1)
assert table.scan().use_ref(branch).to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
{"id": 1, "value": "updated", "note": None},
{"id": 2, "value": "two", "note": None},
{"id": 3, "value": "inserted", "note": None},
]


def test_upsert_without_new_optional_struct_field(catalog: Catalog) -> None:
original = pa.table({"id": [1, 2], "payload": [{"value": "one"}, {"value": "two"}]})
table = catalog.create_table("default.test_upsert_without_new_optional_struct_field", original.schema)
table.append(original)
table.update_schema().add_column(("payload", "note"), StringType()).commit()

source = pa.table({"id": [1, 2, 3], "payload": [{"value": "updated"}, {"value": "two"}, {"value": "inserted"}]})
result = table.upsert(source, join_cols=["id"])

assert_upsert_result(result, expected_updated=1, expected_inserted=1)
assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
{"id": 1, "payload": {"value": "updated", "note": None}},
{"id": 2, "payload": {"value": "two", "note": None}},
{"id": 3, "payload": {"value": "inserted", "note": None}},
]


def test_upsert_matches_renamed_struct_fields_by_id(catalog: Catalog) -> None:
original = pa.table({"id": [1], "payload": [{"a": 1, "b": 2}]})
table = catalog.create_table("default.test_upsert_matches_renamed_struct_fields_by_id", original.schema)
table.append(original)
original_schema = table.schema().as_arrow()
with table.update_schema() as update_schema:
update_schema.rename_column(("payload", "a"), "b")
update_schema.rename_column(("payload", "b"), "a")

source = pa.Table.from_pylist([{"id": 1, "payload": {"a": 2, "b": 1}}], schema=original_schema)
result = table.upsert(source, join_cols=["id"])

assert_upsert_result(result, expected_updated=1, expected_inserted=0)
assert table.scan().to_arrow().to_pylist() == [{"id": 1, "payload": {"b": 2, "a": 1}}]
assert_upsert_result(table.upsert(source, join_cols=["id"]), expected_updated=0, expected_inserted=0)


def test_upsert_matches_renamed_column_by_id(catalog: Catalog) -> None:
original = pa.table({"id": [1], "value": ["old"]})
table = catalog.create_table("default.test_upsert_matches_renamed_column_by_id", original.schema)
table.append(original)
original_schema = table.schema().as_arrow()
table.update_schema().rename_column("value", "renamed").commit()

source = pa.Table.from_pylist([{"id": 1, "value": "updated"}], schema=original_schema)
result = table.upsert(source, join_cols=["id"])

assert_upsert_result(result, expected_updated=1, expected_inserted=0)
assert table.scan().to_arrow().to_pylist() == [{"id": 1, "renamed": "updated"}]


def test_upsert_ignores_dropped_field_with_reused_name(catalog: Catalog) -> None:
original = pa.table({"id": [1, 2], "value": ["old one", "old two"], "label": ["one", "two"]})
table = catalog.create_table("default.test_upsert_ignores_dropped_field_with_reused_name", original.schema)
table.append(original)
original_schema = table.schema()
table.update_schema().delete_column("value").commit()
table.update_schema().add_column("value", StringType()).commit()
assert table.schema().find_field("value").field_id != original_schema.find_field("value").field_id

source = pa.Table.from_pylist(
[
{"id": 1, "value": "stale update", "label": "updated"},
{"id": 2, "value": "stale unchanged", "label": "two"},
{"id": 3, "value": "stale insert", "label": "inserted"},
],
schema=original_schema.as_arrow(),
)
result = table.upsert(source, join_cols=["id"])

assert_upsert_result(result, expected_updated=1, expected_inserted=1)
assert table.scan().to_arrow().sort_by([("id", "ascending")]).to_pylist() == [
{"id": 1, "label": "updated", "value": None},
{"id": 2, "label": "two", "value": None},
{"id": 3, "label": "inserted", "value": None},
]


def test_upsert_with_nulls(catalog: Catalog) -> None:
identifier = "default.test_upsert_with_nulls"
_drop_table(catalog, identifier)
Expand Down
Loading