Skip to content

Commit 710d15e

Browse files
rahulsmahadevIsaac
andauthored
Standardize uuid values on uuid.UUID to fix uuid partition filters (#4370)
* Plan uuid identity partitions with the literal's uuid representation Manifests are read with uuid partition values as uuid.UUID, while UUID literals hold the 16 bytes, so partition filters and residuals on a uuid identity partition never matched: EqualTo and In planned no files, and NotEqualTo and NotIn planned every file. Give partition values the literal representation before the partition and residual evaluators compare them. Co-authored-by: Isaac <no-reply@databricks.com> * Represent uuid values as uuid.UUID in literals and conversions The Avro reader returns uuid.UUID for uuid values, while UUIDLiteral and from_bytes(UUIDType) used the 16 bytes, so the partition and residual evaluators, and the projected identity partition values, compared a UUID with bytes. Standardize on uuid.UUID, as Java does (Literal<UUID>, Conversions.fromByteBuffer): UUIDLiteral wraps a UUID, the string, fixed and binary conversions build one, from_bytes(UUIDType) returns one, and identity partition values stay UUID. PyArrow gets the 16 bytes. This replaces the partition value conversion in the scan planner. Co-authored-by: Isaac <no-reply@databricks.com> --------- Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 2dd390b commit 710d15e

8 files changed

Lines changed: 176 additions & 36 deletions

File tree

‎pyiceberg/conversions.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -388,13 +388,17 @@ def _(_: StringType, b: bytes) -> str:
388388

389389
@from_bytes.register(BinaryType)
390390
@from_bytes.register(FixedType)
391-
@from_bytes.register(UUIDType)
392391
@from_bytes.register(GeometryType)
393392
@from_bytes.register(GeographyType)
394393
def _(_: PrimitiveType, b: bytes) -> bytes:
395394
return b
396395

397396

397+
@from_bytes.register(UUIDType)
398+
def _(_: UUIDType, b: bytes) -> uuid.UUID:
399+
return uuid.UUID(bytes=b)
400+
401+
398402
@from_bytes.register(DecimalType)
399403
def _(primitive_type: DecimalType, buf: bytes) -> Decimal:
400404
unscaled = int.from_bytes(buf, "big", signed=True)

‎pyiceberg/expressions/literals.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ def literal(value: L) -> Literal[L]:
165165
elif isinstance(value, str):
166166
return StringLiteral(value)
167167
elif isinstance(value, UUID):
168-
return UUIDLiteral(value.bytes)
168+
return UUIDLiteral(value)
169169
elif isinstance(value, bytes):
170170
return BinaryLiteral(value)
171171
elif isinstance(value, Decimal):
@@ -640,8 +640,8 @@ def _(self, _: TimestamptzType) -> Literal[int]:
640640
return TimestampLiteral(timestamptz_to_micros(self.value))
641641

642642
@to.register(UUIDType)
643-
def _(self, _: UUIDType) -> Literal[bytes]:
644-
return UUIDLiteral(UUID(self.value).bytes)
643+
def _(self, _: UUIDType) -> Literal[UUID]:
644+
return UUIDLiteral(UUID(self.value))
645645

646646
@to.register(DecimalType)
647647
def _(self, type_var: DecimalType) -> Literal[Decimal]:
@@ -685,34 +685,34 @@ def __repr__(self) -> str:
685685
return f"literal({repr(self.value)})"
686686

687687

688-
class UUIDLiteral(Literal[bytes]):
689-
def __init__(self, value: bytes) -> None:
690-
super().__init__(value, bytes)
688+
class UUIDLiteral(Literal[UUID]):
689+
def __init__(self, value: UUID) -> None:
690+
super().__init__(value, UUID)
691691

692692
@model_serializer
693693
def ser_model(self) -> UUID:
694-
return UUID(bytes=self.root)
694+
return self.root
695695

696696
@singledispatchmethod
697697
def to(self, type_var: IcebergType) -> Literal: # type: ignore
698698
raise TypeError(f"Cannot convert UUIDLiteral into {type_var}")
699699

700700
@to.register(UUIDType)
701-
def _(self, _: UUIDType) -> Literal[bytes]:
701+
def _(self, _: UUIDType) -> Literal[UUID]:
702702
return self
703703

704704
@to.register(FixedType)
705705
def _(self, type_var: FixedType) -> Literal[bytes]:
706706
if len(type_var) == UUID_BYTES_LENGTH:
707-
return FixedLiteral(self.value)
707+
return FixedLiteral(self.value.bytes)
708708
else:
709709
raise TypeError(
710710
f"Cannot convert UUIDLiteral into {type_var}, different length: {len(type_var)} <> {UUID_BYTES_LENGTH}"
711711
)
712712

713713
@to.register(BinaryType)
714714
def _(self, _: BinaryType) -> Literal[bytes]:
715-
return BinaryLiteral(self.value)
715+
return BinaryLiteral(self.value.bytes)
716716

717717

718718
class FixedLiteral(Literal[bytes]):
@@ -741,9 +741,9 @@ def _(self, _: BinaryType) -> Literal[bytes]:
741741
return BinaryLiteral(self.value)
742742

743743
@to.register(UUIDType)
744-
def _(self, type_var: UUIDType) -> Literal[bytes]:
744+
def _(self, type_var: UUIDType) -> Literal[UUID]:
745745
if len(self.value) == UUID_BYTES_LENGTH:
746-
return UUIDLiteral(self.value)
746+
return UUIDLiteral(UUID(bytes=self.value))
747747
else:
748748
raise TypeError(
749749
f"Could not convert {self.value!r} into a {type_var}, lengths differ {len(self.value)} <> {UUID_BYTES_LENGTH}"
@@ -776,9 +776,9 @@ def _(self, type_var: FixedType) -> Literal[bytes]:
776776
)
777777

778778
@to.register(UUIDType)
779-
def _(self, type_var: UUIDType) -> Literal[bytes]:
779+
def _(self, type_var: UUIDType) -> Literal[UUID]:
780780
if len(self.value) == UUID_BYTES_LENGTH:
781-
return UUIDLiteral(self.value)
781+
return UUIDLiteral(UUID(bytes=self.value))
782782
else:
783783
raise TypeError(
784784
f"Cannot convert BinaryLiteral into {type_var}, different length: {UUID_BYTES_LENGTH} <> {len(self.value)}"

‎pyiceberg/io/pyarrow.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -897,10 +897,15 @@ def _literal_arrow_type(iceberg_type: IcebergType, values: Iterable[Any]) -> pa.
897897
return visit(iceberg_type, _ConvertToArrowSchema())
898898

899899

900+
def _to_arrow_value(value: Any) -> Any:
901+
# PyArrow builds uuid values from their 16 bytes; older PyArrow releases reject a uuid.UUID
902+
return value.bytes if isinstance(value, uuid.UUID) else value
903+
904+
900905
def _convert_scalar(value: Any, iceberg_type: IcebergType) -> pa.scalar:
901906
if not isinstance(iceberg_type, PrimitiveType):
902907
raise ValueError(f"Expected primitive type, got: {iceberg_type}")
903-
return pa.scalar(value=value, type=_literal_arrow_type(iceberg_type, [value]))
908+
return pa.scalar(value=_to_arrow_value(value), type=_literal_arrow_type(iceberg_type, [value]))
904909

905910

906911
class _ConvertToArrowExpression(BoundBooleanExpressionVisitor[pc.Expression]):
@@ -937,11 +942,15 @@ def _get_field_name(self, term: BoundTerm) -> str | tuple[str, ...]:
937942
return term.ref().field.name
938943

939944
def visit_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression:
940-
pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals))
945+
pyarrow_literals = pa.array(
946+
[_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals)
947+
)
941948
return pc.field(self._get_field_name(term)).isin(pyarrow_literals)
942949

943950
def visit_not_in(self, term: BoundTerm, literals: set[Any]) -> pc.Expression:
944-
pyarrow_literals = pa.array(literals, type=_literal_arrow_type(term.ref().field.field_type, literals))
951+
pyarrow_literals = pa.array(
952+
[_to_arrow_value(literal) for literal in literals], type=_literal_arrow_type(term.ref().field.field_type, literals)
953+
)
945954
return ~pc.field(self._get_field_name(term)).isin(pyarrow_literals)
946955

947956
def visit_is_nan(self, term: BoundTerm) -> pc.Expression:

‎pyiceberg/partitioning.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema:
487487
Return the Partition Record representation of the value.
488488
489489
The value is first converted to internal partition representation.
490-
For example, UUID is converted to bytes[16], DateType to days since epoch, etc.
490+
For example, UUID values become uuid.UUID, and DateType values become days since epoch.
491491
492492
Then the corresponding PartitionField's transform is applied to return
493493
the final partition record value.
@@ -498,7 +498,7 @@ def partition_record_value(partition_field: PartitionField, value: Any, schema:
498498

499499
@singledispatch
500500
def _to_partition_representation(type: IcebergType, value: Any) -> Any:
501-
"""Strip the logical type into the physical type.
501+
"""Return the internal partition representation of a value.
502502
503503
It can be that the value is already transformed into its physical type,
504504
in this case it will return the original value. Keep in mind that the
@@ -560,13 +560,13 @@ def _(type: IcebergType, value: int | time | None) -> int | None:
560560

561561

562562
@_to_partition_representation.register(UUIDType)
563-
def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> bytes | int | None:
563+
def _(type: IcebergType, value: uuid.UUID | int | bytes | None) -> uuid.UUID | int | None:
564564
if value is None:
565565
return None
566566
elif isinstance(value, bytes):
567-
return value # IdentityTransform
567+
return uuid.UUID(bytes=value) # IdentityTransform
568568
elif isinstance(value, uuid.UUID):
569-
return value.bytes # IdentityTransform
569+
return value # IdentityTransform
570570
elif isinstance(value, int):
571571
return value # BucketTransform
572572
else:

‎tests/expressions/test_literals.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@ def test_string_to_uuid_literal() -> None:
410410
uuid_str = literal(str(expected))
411411
uuid_lit = uuid_str.to(UUIDType())
412412

413-
assert expected.bytes == uuid_lit.value
413+
assert expected == uuid_lit.value
414414

415415

416416
def test_string_to_decimal_literal() -> None:
@@ -567,8 +567,8 @@ def test_binary_to_uuid() -> None:
567567
lit = literal(test_uuid.bytes)
568568
uuid_lit = lit.to(UUIDType())
569569
assert uuid_lit is not None
570-
assert lit.value == uuid_lit.value
571-
assert uuid_lit.value == test_uuid.bytes
570+
assert lit.value == test_uuid.bytes
571+
assert uuid_lit.value == test_uuid
572572

573573

574574
def test_incompatible_binary_to_uuid() -> None:
@@ -597,8 +597,8 @@ def test_fixed_to_uuid() -> None:
597597
lit = literal(test_uuid.bytes).to(FixedType(16))
598598
uuid_lit = lit.to(UUIDType())
599599
assert uuid_lit is not None
600-
assert lit.value == uuid_lit.value
601-
assert uuid_lit.value == test_uuid.bytes
600+
assert lit.value == test_uuid.bytes
601+
assert uuid_lit.value == test_uuid
602602

603603

604604
def test_incompatible_fixed_to_uuid() -> None:
@@ -1057,7 +1057,7 @@ def test_uuid_literal_initialization() -> None:
10571057
test_uuid = uuid.UUID("f79c3e09-677c-4bbd-a479-3f349cb785e7")
10581058
uuid_literal = literal(test_uuid)
10591059
assert isinstance(uuid_literal, Literal)
1060-
assert test_uuid.bytes == uuid_literal.value
1060+
assert test_uuid == uuid_literal.value
10611061

10621062

10631063
def test_uuid_to_fixed() -> None:

‎tests/io/test_pyarrow.py‎

Lines changed: 119 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,25 @@ def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference) -> Non
915915
)
916916

917917

918+
def test_uuid_expressions_to_pyarrow() -> None:
919+
schema = Schema(NestedField(1, "u", UUIDType(), required=False))
920+
ref = BoundReference(schema.find_field(1), schema.accessor_for_field(1))
921+
first, second = uuid.UUID(int=1), uuid.UUID(int=2)
922+
literals = {literal(first), literal(second)}
923+
# PyArrow builds uuid values from their 16 bytes, so compare with expressions built from bytes in either set order
924+
expected_in = [
925+
ds.field("u").isin(pa.array(values, type=pa.uuid()))
926+
for values in ([first.bytes, second.bytes], [second.bytes, first.bytes])
927+
]
928+
for expression, expected in [
929+
(BoundEqualTo(ref, literal(first)), [ds.field("u") == pa.scalar(first.bytes, type=pa.uuid())]),
930+
(BoundIn(ref, literals), expected_in),
931+
(BoundNotIn(ref, literals), [~candidate for candidate in expected_in]),
932+
]:
933+
actual = expression_to_pyarrow(expression)
934+
assert any(actual.equals(candidate) for candidate in expected), expression
935+
936+
918937
def test_and_to_pyarrow(bound_reference: BoundReference) -> None:
919938
assert (
920939
repr(expression_to_pyarrow(And(BoundEqualTo(bound_reference, literal("hello")), BoundIsNull(bound_reference))))
@@ -1571,6 +1590,53 @@ def test_identity_transform_column_projection_with_falsy_value(
15711590
)
15721591

15731592

1593+
def test_identity_transform_column_projection_uuid(tmp_path: str, catalog: InMemoryCatalog) -> None:
1594+
"""A filter on a uuid identity partition column matches the files that do not contain the column."""
1595+
schema = Schema(
1596+
NestedField(1, "other_field", StringType(), required=False),
1597+
NestedField(2, "partition_col", UUIDType(), required=False),
1598+
)
1599+
partition_spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "partition_col"))
1600+
catalog.create_namespace("default")
1601+
table = catalog.create_table(
1602+
"default.test_projection_partition_uuid",
1603+
schema=schema,
1604+
partition_spec=partition_spec,
1605+
properties={TableProperties.DEFAULT_NAME_MAPPING: create_mapping_from_schema(schema).model_dump_json()},
1606+
)
1607+
1608+
file_loc = f"{tmp_path}/test.parquet"
1609+
pq.write_table(pa.table([pa.array(["foo", "bar"], type=pa.string())], names=["other_field"]), file_loc)
1610+
statistics = data_file_statistics_from_parquet_metadata(
1611+
parquet_metadata=pq.read_metadata(file_loc),
1612+
stats_columns=compute_statistics_plan(table.schema(), table.metadata.properties),
1613+
parquet_column_mapping=parquet_path_to_id_mapping(table.schema()),
1614+
)
1615+
partition_value = uuid.UUID(int=1)
1616+
data_file = DataFile.from_args(
1617+
content=DataFileContent.DATA,
1618+
file_path=file_loc,
1619+
file_format=FileFormat.PARQUET,
1620+
partition=Record(partition_value),
1621+
file_size_in_bytes=os.path.getsize(file_loc),
1622+
sort_order_id=None,
1623+
spec_id=table.metadata.default_spec_id,
1624+
equality_ids=None,
1625+
key_metadata=None,
1626+
**statistics.to_serialized_dict(),
1627+
)
1628+
with table.transaction() as transaction:
1629+
with transaction.update_snapshot().overwrite() as update:
1630+
update.append_data_file(data_file)
1631+
1632+
for row_filter, expected in [
1633+
(f"partition_col = '{partition_value}'", ["foo", "bar"]),
1634+
(f"partition_col != '{partition_value}'", []),
1635+
]:
1636+
result = table.scan(row_filter=row_filter, selected_fields=("other_field",)).to_arrow()
1637+
assert result["other_field"].to_pylist() == expected, row_filter
1638+
1639+
15741640
def test_identity_transform_null_partition_value_takes_precedence_over_initial_default(tmpdir: str) -> None:
15751641
"""A null identity partition value must project as null, not as the field's initial-default."""
15761642
table_schema = Schema(
@@ -3518,9 +3584,9 @@ def test_determine_partitions_with_uuid_column() -> None:
35183584

35193585
partitions = list(_determine_partitions(spec, schema, arrow_table))
35203586

3521-
# The partition values are the 16 bytes of each UUID
3587+
# The partition values are uuid.UUID, as they are read back from the manifests
35223588
rows_by_partition = {p.partition_key.partition[0]: sorted(p.arrow_table_partition["id"].to_pylist()) for p in partitions}
3523-
assert rows_by_partition == {uuid.UUID(int=1).bytes: [1, 3], uuid.UUID(int=2).bytes: [2], None: [4]}
3589+
assert rows_by_partition == {uuid.UUID(int=1): [1, 3], uuid.UUID(int=2): [2], None: [4]}
35243590
# The written data keeps the uuid type
35253591
assert all(p.arrow_table_partition.schema.field("uuid").type == pa.uuid() for p in partitions)
35263592

@@ -3546,6 +3612,57 @@ def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> N
35463612
assert {task.file.partition[0] for task in table.scan().plan_files()} == {uuid.UUID(int=1), uuid.UUID(int=2), None}
35473613

35483614

3615+
def test_scan_plans_uuid_identity_partitions(catalog: InMemoryCatalog) -> None:
3616+
schema = Schema(
3617+
NestedField(id=1, name="id", field_type=LongType(), required=False),
3618+
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
3619+
)
3620+
spec = PartitionSpec(PartitionField(source_id=2, field_id=1000, transform=IdentityTransform(), name="uuid"))
3621+
catalog.create_namespace("default")
3622+
table = catalog.create_table("default.test_scan_uuid_partitioned", schema=schema, partition_spec=spec)
3623+
first, second, third = uuid.UUID(int=1), uuid.UUID(int=2), uuid.UUID(int=3)
3624+
table.append(
3625+
pa.table(
3626+
{
3627+
"id": pa.array([1, 2, 3], type=pa.int64()),
3628+
"uuid": pa.array([first.bytes, second.bytes, third.bytes], type=pa.uuid()),
3629+
}
3630+
)
3631+
)
3632+
3633+
# UUID partition values read from manifests must compare equal to bound UUID literals
3634+
for row_filter, expected in [
3635+
(f"uuid = '{first}'", {first}),
3636+
(f"uuid in ('{first}', '{second}')", {first, second}),
3637+
(f"uuid != '{first}'", {second, third}),
3638+
(f"uuid not in ('{first}', '{second}')", {third}),
3639+
]:
3640+
scan = table.scan(row_filter=row_filter)
3641+
assert {task.file.partition[0] for task in scan.plan_files()} == expected, row_filter
3642+
assert scan.count() == len(expected), row_filter
3643+
3644+
3645+
def test_scan_prunes_uuid_files_by_bounds(catalog: InMemoryCatalog) -> None:
3646+
schema = Schema(
3647+
NestedField(id=1, name="id", field_type=LongType(), required=False),
3648+
NestedField(id=2, name="uuid", field_type=UUIDType(), required=False),
3649+
)
3650+
catalog.create_namespace("default")
3651+
table = catalog.create_table("default.test_scan_uuid_bounds", schema=schema)
3652+
for ids in ([1, 2], [3, 4]):
3653+
values = [uuid.UUID(int=value).bytes for value in ids]
3654+
table.append(pa.table({"id": pa.array(ids, type=pa.int64()), "uuid": pa.array(values, type=pa.uuid())}))
3655+
3656+
# The lower and upper bounds of the uuid column are compared with the literal to prune files
3657+
for row_filter, expected_files in [
3658+
(f"uuid = '{uuid.UUID(int=1)}'", 1),
3659+
(f"uuid > '{uuid.UUID(int=2)}'", 1),
3660+
(f"uuid < '{uuid.UUID(int=5)}'", 2),
3661+
(f"uuid in ('{uuid.UUID(int=1)}', '{uuid.UUID(int=4)}')", 2),
3662+
]:
3663+
assert len(list(table.scan(row_filter=row_filter).plan_files())) == expected_files, row_filter
3664+
3665+
35493666
def test_append_view_types_to_partitioned_table(catalog: InMemoryCatalog) -> None:
35503667
schema = Schema(
35513668
NestedField(id=1, name="id", field_type=LongType(), required=False),

‎tests/table/test_partitioning.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import pytest
2323

2424
from pyiceberg.exceptions import ValidationError
25-
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec
25+
from pyiceberg.partitioning import UNPARTITIONED_PARTITION_SPEC, PartitionField, PartitionSpec, partition_record_value
2626
from pyiceberg.schema import Schema
2727
from pyiceberg.transforms import (
2828
BucketTransform,
@@ -220,6 +220,16 @@ def test_partition_type_missing_source_field(table_schema_simple: Schema) -> Non
220220
)
221221

222222

223+
def test_uuid_identity_partition_record_value_and_path() -> None:
224+
schema = Schema(NestedField(field_id=1, name="u", field_type=UUIDType(), required=False))
225+
spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="u"))
226+
value = UUID(int=1)
227+
for raw in (value, value.bytes):
228+
assert partition_record_value(spec.fields[0], raw, schema) == value, raw
229+
# The partition path holds the uuid string, as in Java
230+
assert spec.partition_to_path(Record(value), schema) == "u=00000000-0000-0000-0000-000000000001"
231+
232+
223233
@pytest.mark.parametrize(
224234
"source_type, value",
225235
[

0 commit comments

Comments
 (0)