Skip to content

Commit 8005b29

Browse files
rahulsmahadevIsaac
andcommitted
Fix append to a table identity-partitioned on a uuid column
Appending to a table identity-partitioned on a uuid column raised ArrowNotImplementedError: _determine_partitions groups and filters on the arrow.uuid extension type, which PyArrow has no kernels for. Use the storage type (fixed_size_binary(16)) of extension-typed partition columns; the written data keeps its uuid type. Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 0008d65 commit 8005b29

2 files changed

Lines changed: 49 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3240,7 +3240,11 @@ def _determine_partitions(spec: PartitionSpec, schema: Schema, arrow_table: pa.T
32403240
if full_field_name is None:
32413241
raise ValueError(f"Could not find column name for field ID: {partition.source_id}")
32423242
field_array = _get_field_from_arrow_table(arrow_table, full_field_name)
3243-
arrow_table = arrow_table.append_column(name, partition.transform.pyarrow_transform(source_field.field_type)(field_array))
3243+
partition_array = partition.transform.pyarrow_transform(source_field.field_type)(field_array)
3244+
# PyArrow has no group by or equality kernels for extension types such as arrow.uuid, so use their storage type
3245+
if isinstance(partition_array.type, pa.BaseExtensionType):
3246+
partition_array = partition_array.cast(partition_array.type.storage_type)
3247+
arrow_table = arrow_table.append_column(name, partition_array)
32443248

32453249
unique_partition_fields = arrow_table.select(partition_fields).group_by(partition_fields).aggregate([])
32463250

‎tests/io/test_pyarrow.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@
137137
TimestampType,
138138
TimestamptzType,
139139
TimeType,
140+
UUIDType,
140141
)
141142
from tests.catalog.test_base import InMemoryCatalog
142143
from tests.conftest import UNIFIED_AWS_SESSION_PROPERTIES
@@ -3471,6 +3472,49 @@ def test_determine_partitions_with_view_types(transform: Transform[Any, Any]) ->
34713472
assert sorted(written.to_pylist(), key=str) == sorted(rows, key=str)
34723473

34733474

3475+
def test_determine_partitions_with_uuid_column() -> None:
3476+
schema = Schema(
3477+
NestedField(id=1, name="id", field_type=LongType(), required=False),
3478+
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
3479+
)
3480+
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid"))
3481+
arrow_table = pa.table(
3482+
{
3483+
"id": pa.array([1, 2, 3, 4], type=pa.int64()),
3484+
"uuid": pa.array([uuid.UUID(int=1).bytes, uuid.UUID(int=2).bytes, uuid.UUID(int=1).bytes, None], type=pa.uuid()),
3485+
}
3486+
)
3487+
3488+
partitions = list(_determine_partitions(spec, schema, arrow_table))
3489+
3490+
# The partition values are the 16 bytes of each UUID
3491+
rows_by_partition = {p.partition_key.partition[0]: sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions}
3492+
assert rows_by_partition == {uuid.UUID(int=1).bytes: [1, 3], uuid.UUID(int=2).bytes: [2], None: [4]}
3493+
# The written data keeps the uuid type
3494+
assert all(p.arrow_table_partition.schema.field("uuid").type == pa.uuid() for p in partitions)
3495+
3496+
3497+
def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> None:
3498+
schema = Schema(
3499+
NestedField(id=1, name="id", field_type=LongType(), required=False),
3500+
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
3501+
)
3502+
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid"))
3503+
catalog.create_namespace("default")
3504+
table = catalog.create_table("default.test_append_uuid_partitioned", schema=schema, partition_spec=spec)
3505+
3506+
arrow_table = pa.table(
3507+
{
3508+
"id": pa.array([1, 2, 3], type=pa.int64()),
3509+
"uuid": pa.array([uuid.UUID(int=1).bytes, uuid.UUID(int=2).bytes, None], type=pa.uuid()),
3510+
}
3511+
)
3512+
table.append(arrow_table)
3513+
3514+
assert sorted(table.scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == arrow_table.to_pylist()
3515+
assert {task.file.partition[0] for task in table.scan().plan_files()} == {uuid.UUID(int=1), uuid.UUID(int=2), None}
3516+
3517+
34743518
def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None:
34753519
schema = Schema(
34763520
NestedField(id=1, name="id", field_type=LongType(), required=False),

0 commit comments

Comments
 (0)