@@ -915,6 +915,25 @@ def test_expr_not_starts_with_to_pyarrow(bound_reference: BoundReference) -> Non
915915 )
916916
917917
918+ def test_uuid_expressions_to_pyarrow () -> None :
919+ schema = Schema (NestedField (1 , "u" , UUIDType (), required = False ))
920+ ref = BoundReference (schema .find_field (1 ), schema .accessor_for_field (1 ))
921+ first , second = uuid .UUID (int = 1 ), uuid .UUID (int = 2 )
922+ literals = {literal (first ), literal (second )}
923+ # PyArrow builds uuid values from their 16 bytes, so compare with expressions built from bytes in either set order
924+ expected_in = [
925+ ds .field ("u" ).isin (pa .array (values , type = pa .uuid ()))
926+ for values in ([first .bytes , second .bytes ], [second .bytes , first .bytes ])
927+ ]
928+ for expression , expected in [
929+ (BoundEqualTo (ref , literal (first )), [ds .field ("u" ) == pa .scalar (first .bytes , type = pa .uuid ())]),
930+ (BoundIn (ref , literals ), expected_in ),
931+ (BoundNotIn (ref , literals ), [~ candidate for candidate in expected_in ]),
932+ ]:
933+ actual = expression_to_pyarrow (expression )
934+ assert any (actual .equals (candidate ) for candidate in expected ), expression
935+
936+
918937def test_and_to_pyarrow (bound_reference : BoundReference ) -> None :
919938 assert (
920939 repr (expression_to_pyarrow (And (BoundEqualTo (bound_reference , literal ("hello" )), BoundIsNull (bound_reference ))))
@@ -1571,6 +1590,53 @@ def test_identity_transform_column_projection_with_falsy_value(
15711590 )
15721591
15731592
1593+ def test_identity_transform_column_projection_uuid (tmp_path : str , catalog : InMemoryCatalog ) -> None :
1594+ """A filter on a uuid identity partition column matches the files that do not contain the column."""
1595+ schema = Schema (
1596+ NestedField (1 , "other_field" , StringType (), required = False ),
1597+ NestedField (2 , "partition_col" , UUIDType (), required = False ),
1598+ )
1599+ partition_spec = PartitionSpec (PartitionField (2 , 1000 , IdentityTransform (), "partition_col" ))
1600+ catalog .create_namespace ("default" )
1601+ table = catalog .create_table (
1602+ "default.test_projection_partition_uuid" ,
1603+ schema = schema ,
1604+ partition_spec = partition_spec ,
1605+ properties = {TableProperties .DEFAULT_NAME_MAPPING : create_mapping_from_schema (schema ).model_dump_json ()},
1606+ )
1607+
1608+ file_loc = f"{ tmp_path } /test.parquet"
1609+ pq .write_table (pa .table ([pa .array (["foo" , "bar" ], type = pa .string ())], names = ["other_field" ]), file_loc )
1610+ statistics = data_file_statistics_from_parquet_metadata (
1611+ parquet_metadata = pq .read_metadata (file_loc ),
1612+ stats_columns = compute_statistics_plan (table .schema (), table .metadata .properties ),
1613+ parquet_column_mapping = parquet_path_to_id_mapping (table .schema ()),
1614+ )
1615+ partition_value = uuid .UUID (int = 1 )
1616+ data_file = DataFile .from_args (
1617+ content = DataFileContent .DATA ,
1618+ file_path = file_loc ,
1619+ file_format = FileFormat .PARQUET ,
1620+ partition = Record (partition_value ),
1621+ file_size_in_bytes = os .path .getsize (file_loc ),
1622+ sort_order_id = None ,
1623+ spec_id = table .metadata .default_spec_id ,
1624+ equality_ids = None ,
1625+ key_metadata = None ,
1626+ ** statistics .to_serialized_dict (),
1627+ )
1628+ with table .transaction () as transaction :
1629+ with transaction .update_snapshot ().overwrite () as update :
1630+ update .append_data_file (data_file )
1631+
1632+ for row_filter , expected in [
1633+ (f"partition_col = '{ partition_value } '" , ["foo" , "bar" ]),
1634+ (f"partition_col != '{ partition_value } '" , []),
1635+ ]:
1636+ result = table .scan (row_filter = row_filter , selected_fields = ("other_field" ,)).to_arrow ()
1637+ assert result ["other_field" ].to_pylist () == expected , row_filter
1638+
1639+
15741640def test_identity_transform_null_partition_value_takes_precedence_over_initial_default (tmpdir : str ) -> None :
15751641 """A null identity partition value must project as null, not as the field's initial-default."""
15761642 table_schema = Schema (
@@ -3518,9 +3584,9 @@ def test_determine_partitions_with_uuid_column() -> None:
35183584
35193585 partitions = list (_determine_partitions (spec , schema , arrow_table ))
35203586
3521- # The partition values are the 16 bytes of each UUID
3587+ # The partition values are uuid.UUID, as they are read back from the manifests
35223588 rows_by_partition = {p .partition_key .partition [0 ]: sorted (p .arrow_table_partition ["id" ].to_pylist ()) for p in partitions }
3523- assert rows_by_partition == {uuid .UUID (int = 1 ). bytes : [1 , 3 ], uuid .UUID (int = 2 ). bytes : [2 ], None : [4 ]}
3589+ assert rows_by_partition == {uuid .UUID (int = 1 ): [1 , 3 ], uuid .UUID (int = 2 ): [2 ], None : [4 ]}
35243590 # The written data keeps the uuid type
35253591 assert all (p .arrow_table_partition .schema .field ("uuid" ).type == pa .uuid () for p in partitions )
35263592
@@ -3546,6 +3612,57 @@ def test_append_uuid_to_table_partitioned_on_uuid(catalog: InMemoryCatalog) -> N
35463612 assert {task .file .partition [0 ] for task in table .scan ().plan_files ()} == {uuid .UUID (int = 1 ), uuid .UUID (int = 2 ), None }
35473613
35483614
3615+ def test_scan_plans_uuid_identity_partitions (catalog : InMemoryCatalog ) -> None :
3616+ schema = Schema (
3617+ NestedField (id = 1 , name = "id" , field_type = LongType (), required = False ),
3618+ NestedField (id = 2 , name = "uuid" , field_type = UUIDType (), required = False ),
3619+ )
3620+ spec = PartitionSpec (PartitionField (source_id = 2 , field_id = 1000 , transform = IdentityTransform (), name = "uuid" ))
3621+ catalog .create_namespace ("default" )
3622+ table = catalog .create_table ("default.test_scan_uuid_partitioned" , schema = schema , partition_spec = spec )
3623+ first , second , third = uuid .UUID (int = 1 ), uuid .UUID (int = 2 ), uuid .UUID (int = 3 )
3624+ table .append (
3625+ pa .table (
3626+ {
3627+ "id" : pa .array ([1 , 2 , 3 ], type = pa .int64 ()),
3628+ "uuid" : pa .array ([first .bytes , second .bytes , third .bytes ], type = pa .uuid ()),
3629+ }
3630+ )
3631+ )
3632+
3633+ # UUID partition values read from manifests must compare equal to bound UUID literals
3634+ for row_filter , expected in [
3635+ (f"uuid = '{ first } '" , {first }),
3636+ (f"uuid in ('{ first } ', '{ second } ')" , {first , second }),
3637+ (f"uuid != '{ first } '" , {second , third }),
3638+ (f"uuid not in ('{ first } ', '{ second } ')" , {third }),
3639+ ]:
3640+ scan = table .scan (row_filter = row_filter )
3641+ assert {task .file .partition [0 ] for task in scan .plan_files ()} == expected , row_filter
3642+ assert scan .count () == len (expected ), row_filter
3643+
3644+
3645+ def test_scan_prunes_uuid_files_by_bounds (catalog : InMemoryCatalog ) -> None :
3646+ schema = Schema (
3647+ NestedField (id = 1 , name = "id" , field_type = LongType (), required = False ),
3648+ NestedField (id = 2 , name = "uuid" , field_type = UUIDType (), required = False ),
3649+ )
3650+ catalog .create_namespace ("default" )
3651+ table = catalog .create_table ("default.test_scan_uuid_bounds" , schema = schema )
3652+ for ids in ([1 , 2 ], [3 , 4 ]):
3653+ values = [uuid .UUID (int = value ).bytes for value in ids ]
3654+ table .append (pa .table ({"id" : pa .array (ids , type = pa .int64 ()), "uuid" : pa .array (values , type = pa .uuid ())}))
3655+
3656+ # The lower and upper bounds of the uuid column are compared with the literal to prune files
3657+ for row_filter , expected_files in [
3658+ (f"uuid = '{ uuid .UUID (int = 1 )} '" , 1 ),
3659+ (f"uuid > '{ uuid .UUID (int = 2 )} '" , 1 ),
3660+ (f"uuid < '{ uuid .UUID (int = 5 )} '" , 2 ),
3661+ (f"uuid in ('{ uuid .UUID (int = 1 )} ', '{ uuid .UUID (int = 4 )} ')" , 2 ),
3662+ ]:
3663+ assert len (list (table .scan (row_filter = row_filter ).plan_files ())) == expected_files , row_filter
3664+
3665+
35493666def test_append_view_types_to_partitioned_table (catalog : InMemoryCatalog ) -> None :
35503667 schema = Schema (
35513668 NestedField (id = 1 , name = "id" , field_type = LongType (), required = False ),
0 commit comments