@@ -4271,6 +4271,184 @@ def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataT
42714271 assert result .column (1 ).to_pylist () == [[], None , [2 , None ]]
42724272
42734273
4274+ def test__to_requested_schema_fixed_size_list_of_structs () -> None :
4275+ # The requested schema renames the struct field, like sanitize_column_names does on write
4276+ file_schema = Schema (
4277+ NestedField (
4278+ 1 ,
4279+ "col_list_with_struct" ,
4280+ ListType (11 , StructType (NestedField (111 , "a b" , IntegerType (), required = False )), element_required = False ),
4281+ required = False ,
4282+ ),
4283+ )
4284+ requested_schema = Schema (
4285+ NestedField (
4286+ 1 ,
4287+ "col_list_with_struct" ,
4288+ ListType (11 , StructType (NestedField (111 , "a_b" , IntegerType (), required = False )), element_required = False ),
4289+ required = False ,
4290+ ),
4291+ )
4292+ arrow_schema = pa .schema ([pa .field ("col_list_with_struct" , pa .list_ (pa .struct ([pa .field ("a b" , pa .int32 ())]), 2 ))])
4293+ values = pa .array (
4294+ [[{"a b" : 1 }, None ], None , [{"a b" : None }, {"a b" : 2 }], [{"a b" : 3 }, {"a b" : 4 }]], type = arrow_schema .field (0 ).type
4295+ )
4296+ batch = pa .RecordBatch .from_arrays ([values ], schema = arrow_schema )
4297+
4298+ result = _to_requested_schema (requested_schema , file_schema , batch )
4299+ assert result .column (0 ).to_pylist () == [[{"a_b" : 1 }, None ], None , [{"a_b" : None }, {"a_b" : 2 }], [{"a_b" : 3 }, {"a_b" : 4 }]]
4300+
4301+ # A sliced batch has to keep the nulls as well
4302+ result = _to_requested_schema (requested_schema , file_schema , batch .slice (1 , 3 ))
4303+ assert result .column (0 ).to_pylist () == [None , [{"a_b" : None }, {"a_b" : 2 }], [{"a_b" : 3 }, {"a_b" : 4 }]]
4304+
4305+
4306+ def test__to_requested_schema_list_of_lists_of_renamed_structs () -> None :
4307+ # The renamed struct fields are only in the projected values, so every list around them has to be rebuilt
4308+ file_struct = StructType (NestedField (4 , "a b" , IntegerType (), required = False ))
4309+ requested_struct = StructType (NestedField (4 , "a_b" , IntegerType (), required = False ))
4310+ file_schema = Schema (
4311+ NestedField (
4312+ 1 ,
4313+ "col" ,
4314+ ListType (2 , ListType (3 , file_struct , element_required = False ), element_required = False ),
4315+ required = False ,
4316+ ),
4317+ )
4318+ requested_schema = Schema (
4319+ NestedField (
4320+ 1 ,
4321+ "col" ,
4322+ ListType (2 , ListType (3 , requested_struct , element_required = False ), element_required = False ),
4323+ required = False ,
4324+ ),
4325+ )
4326+ struct_type = pa .struct ([pa .field ("a b" , pa .int32 ())])
4327+ builders : dict [str , Callable [[pa .DataType ], pa .DataType ]] = {
4328+ "list" : pa .list_ ,
4329+ "large_list" : pa .large_list ,
4330+ "fixed_size_list" : lambda value_type : pa .list_ (value_type , 2 ),
4331+ }
4332+ rows = [[[{"a b" : 1 }, None ], [{"a b" : 2 }, {"a b" : None }]], None , [None , [{"a b" : 3 }, {"a b" : 4 }]]]
4333+ expected = [[[{"a_b" : 1 }, None ], [{"a_b" : 2 }, {"a_b" : None }]], None , [None , [{"a_b" : 3 }, {"a_b" : 4 }]]]
4334+ for outer_builder in builders .values ():
4335+ for inner_builder in builders .values ():
4336+ arrow_type = outer_builder (inner_builder (struct_type ))
4337+ batch = pa .RecordBatch .from_arrays ([pa .array (rows , type = arrow_type )], names = ["col" ])
4338+
4339+ result = _to_requested_schema (requested_schema , file_schema , batch )
4340+ assert result .column (0 ).to_pylist () == expected
4341+
4342+ # A sliced batch has to keep the nulls as well, also when the slice is empty
4343+ result = _to_requested_schema (requested_schema , file_schema , batch .slice (1 , 2 ))
4344+ assert result .column (0 ).to_pylist () == expected [1 :]
4345+ assert _to_requested_schema (requested_schema , file_schema , batch .slice (1 , 0 )).num_rows == 0
4346+
4347+ # Control: a fixed-size list of integers goes through the same projection without any struct
4348+ ints_schema = Schema (NestedField (1 , "col" , ListType (2 , IntegerType (), element_required = False ), required = False ))
4349+ ints = pa .RecordBatch .from_arrays ([pa .array ([[1 , 2 ], None , [None , 4 ]], pa .list_ (pa .int32 (), 2 ))], names = ["col" ])
4350+ assert _to_requested_schema (ints_schema , ints_schema , ints ).column (0 ).to_pylist () == [[1 , 2 ], None , [None , 4 ]]
4351+
4352+
4353+ def test_append_fixed_size_list_of_structs (catalog : InMemoryCatalog ) -> None :
4354+ arrow_table = pa .table (
4355+ {
4356+ "id" : pa .array ([1 , 2 ], pa .int64 ()),
4357+ "v" : pa .array (
4358+ [[{"a b" : 1 , "c" : "x" }, {"a b" : 2 , "c" : "y" }], [{"a b" : None , "c" : "z" }, None ]],
4359+ pa .list_ (pa .struct ([("a b" , pa .int32 ()), ("c" , pa .string ())]), 2 ),
4360+ ),
4361+ }
4362+ )
4363+ catalog .create_namespace ("default" )
4364+ table = catalog .create_table ("default.test_append_fixed_size_list_of_structs" , schema = arrow_table .schema )
4365+
4366+ table .append (arrow_table )
4367+
4368+ assert table .scan ().to_arrow ().to_pylist () == arrow_table .to_pylist ()
4369+
4370+
4371+ def test__to_requested_schema_map_of_lists_of_renamed_structs () -> None :
4372+ # The renamed struct fields are in the list values of a map, so the map has to keep the projected values
4373+ file_schema = Schema (
4374+ NestedField (
4375+ 1 ,
4376+ "col" ,
4377+ MapType (
4378+ 2 ,
4379+ StringType (),
4380+ 3 ,
4381+ ListType (4 , StructType (NestedField (5 , "a b" , IntegerType (), required = False )), element_required = False ),
4382+ value_required = False ,
4383+ ),
4384+ required = False ,
4385+ ),
4386+ )
4387+ requested_schema = Schema (
4388+ NestedField (
4389+ 1 ,
4390+ "col" ,
4391+ MapType (
4392+ 2 ,
4393+ StringType (),
4394+ 3 ,
4395+ ListType (4 , StructType (NestedField (5 , "a_b" , IntegerType (), required = False )), element_required = False ),
4396+ value_required = False ,
4397+ ),
4398+ required = False ,
4399+ ),
4400+ )
4401+ struct_type = pa .struct ([pa .field ("a b" , pa .int32 ())])
4402+ builders : list [Callable [[pa .DataType ], pa .DataType ]] = [pa .list_ , pa .large_list , lambda value_type : pa .list_ (value_type , 2 )]
4403+ rows = [[("k" , [{"a b" : 1 }, None ])], None , [("j" , None ), ("m" , [{"a b" : 2 }, {"a b" : 3 }])]]
4404+ expected = [[("k" , [{"a_b" : 1 }, None ])], None , [("j" , None ), ("m" , [{"a_b" : 2 }, {"a_b" : 3 }])]]
4405+ for builder in builders :
4406+ batch = pa .RecordBatch .from_arrays ([pa .array (rows , type = pa .map_ (pa .string (), builder (struct_type )))], names = ["col" ])
4407+
4408+ result = _to_requested_schema (requested_schema , file_schema , batch )
4409+ assert result .column (0 ).to_pylist () == expected
4410+
4411+ # A sliced batch has to keep the null maps and the null list values as well
4412+ result = _to_requested_schema (requested_schema , file_schema , batch .slice (1 , 2 ))
4413+ assert result .column (0 ).to_pylist () == expected [1 :]
4414+
4415+
4416+ def test_append_map_of_fixed_size_list_of_structs (catalog : InMemoryCatalog ) -> None :
4417+ arrow_table = pa .table (
4418+ {
4419+ "v" : pa .array (
4420+ [[("k" , [{"a b" : 1 }, {"a b" : 2 }])]],
4421+ pa .map_ (pa .string (), pa .list_ (pa .struct ([("a b" , pa .int32 ())]), 2 )),
4422+ ),
4423+ }
4424+ )
4425+ catalog .create_namespace ("default" )
4426+ table = catalog .create_table ("default.test_append_map_of_fixed_size_list_of_structs" , schema = arrow_table .schema )
4427+
4428+ table .append (arrow_table )
4429+
4430+ assert table .scan ().to_arrow ().to_pylist () == arrow_table .to_pylist ()
4431+
4432+
4433+ def test_append_map_with_struct_keys (catalog : InMemoryCatalog ) -> None :
4434+ # A struct key without nulls must not get a validity bitmap, or Arrow aborts on the rebuilt map
4435+ # (https://github.com/apache/arrow/issues/51029)
4436+ rows = [[({"a" : 1 }, 2 )], None , [], [({"a" : None }, 3 )]]
4437+ arrow_table = pa .table ({"v" : pa .array (rows , pa .map_ (pa .struct ([("a" , pa .int32 ())]), pa .int32 ()))})
4438+ key_type = StructType (NestedField (4 , "a" , IntegerType (), required = False ))
4439+ schema = Schema (NestedField (1 , "v" , MapType (2 , key_type , 3 , IntegerType (), value_required = False ), required = False ))
4440+ batch = arrow_table .to_batches ()[0 ]
4441+ for projected in (batch , batch .slice (1 , 3 )):
4442+ assert _to_requested_schema (schema , schema , projected ).column (0 ).to_pylist () == projected .column (0 ).to_pylist ()
4443+
4444+ catalog .create_namespace ("default" )
4445+ table = catalog .create_table ("default.test_append_map_with_struct_keys" , schema = arrow_table .schema )
4446+
4447+ table .append (arrow_table )
4448+
4449+ assert table .scan ().to_arrow ().to_pylist () == arrow_table .to_pylist ()
4450+
4451+
42744452@skip_if_pyarrow_too_old
42754453@pytest .mark .parametrize ("scheme" , ["abfs" , "abfss" , "wasb" , "wasbs" ])
42764454def test_adls_account_name_from_location (scheme : str ) -> None :
0 commit comments