Skip to content

Commit 5710c75

Browse files
rahulsmahadevIsaac
andcommitted
Fix fixed-size lists and renamed struct fields in nested lists and maps
The list visitor rebuilt struct lists from their first two buffers, which are not offsets for a FixedSizeListArray, so the append failed. Rebuild every list and map around its projected children, keeping only the buffers its type has, so renamed struct fields inside nested lists and maps are no longer dropped. Project a struct without a validity mask when it has no nulls, so a rebuilt map with struct keys stays valid (apache/arrow#51029). Co-authored-by: Isaac <no-reply@databricks.com>
1 parent 7c425ed commit 5710c75

2 files changed

Lines changed: 209 additions & 29 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2241,7 +2241,8 @@ def struct(
22412241
return pa.StructArray.from_arrays(
22422242
arrays=field_arrays,
22432243
fields=pa.struct(fields),
2244-
mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) else None,
2244+
# Avoid unknown null counts for non-null map keys: https://github.com/apache/arrow/issues/51029
2245+
mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) and struct_array.null_count else None,
22452246
)
22462247

22472248
def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None:
@@ -2250,18 +2251,23 @@ def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array |
22502251
def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None:
22512252
if isinstance(list_array, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) and value_array is not None:
22522253
list_initializer = pa.large_list if isinstance(list_array, pa.LargeListArray) else pa.list_
2253-
if isinstance(value_array, pa.StructArray):
2254-
# This can be removed once this has been fixed:
2255-
# https://github.com/apache/arrow/issues/38809
2256-
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists
2257-
list_array = pa.Array.from_buffers(
2258-
list_initializer(value_array.type),
2259-
len(list_array),
2260-
list_array.buffers()[:2],
2261-
list_array.null_count,
2262-
list_array.offset,
2263-
[value_array],
2264-
)
2254+
# Rebuild the list around the projected values, so the cast below does not cast the unprojected children.
2255+
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists.
2256+
# A fixed-size list has no offsets buffer, so keep its validity buffer and list size instead.
2257+
if isinstance(list_array, pa.FixedSizeListArray):
2258+
arrow_list_type = pa.list_(value_array.type, list_array.type.list_size)
2259+
buffers = list_array.buffers()[:1]
2260+
else:
2261+
arrow_list_type = list_initializer(value_array.type)
2262+
buffers = list_array.buffers()[:2]
2263+
list_array = pa.Array.from_buffers(
2264+
arrow_list_type,
2265+
len(list_array),
2266+
buffers,
2267+
list_array.null_count,
2268+
list_array.offset,
2269+
[value_array],
2270+
)
22652271
value_array = self._cast_if_needed(list_type.element_field, value_array)
22662272
arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type))
22672273
return list_array.cast(arrow_field)
@@ -2278,22 +2284,18 @@ def map(
22782284
self._construct_field(map_type.key_field, key_result.type),
22792285
self._construct_field(map_type.value_field, value_result.type),
22802286
)
2281-
if isinstance(value_result, pa.StructArray):
2282-
# Arrow does not allow reordering of fields, therefore we have to copy the array :(
2283-
entries = pa.StructArray.from_arrays(
2284-
[key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field]
2285-
)
2286-
# Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps
2287-
return pa.Array.from_buffers(
2288-
arrow_field,
2289-
len(map_array),
2290-
map_array.buffers()[:2],
2291-
map_array.null_count,
2292-
map_array.offset,
2293-
[entries],
2294-
)
2295-
else:
2296-
return map_array.cast(arrow_field)
2287+
entries = pa.StructArray.from_arrays(
2288+
[key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field]
2289+
)
2290+
# Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps
2291+
return pa.Array.from_buffers(
2292+
arrow_field,
2293+
len(map_array),
2294+
map_array.buffers()[:2],
2295+
map_array.null_count,
2296+
map_array.offset,
2297+
[entries],
2298+
)
22972299
else:
22982300
return None
22992301

‎tests/io/test_pyarrow.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4271,6 +4271,184 @@ def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataT
42714271
assert result.column(1).to_pylist() == [[], None, [2, None]]
42724272

42734273

4274+
def test__to_requested_schema_fixed_size_list_of_structs() -> None:
4275+
# The requested schema renames the struct field, like sanitize_column_names does on write
4276+
file_schema = Schema(
4277+
NestedField(
4278+
1,
4279+
"col_list_with_struct",
4280+
ListType(11, StructType(NestedField(111, "a b", IntegerType(), required=False)), element_required=False),
4281+
required=False,
4282+
),
4283+
)
4284+
requested_schema = Schema(
4285+
NestedField(
4286+
1,
4287+
"col_list_with_struct",
4288+
ListType(11, StructType(NestedField(111, "a_b", IntegerType(), required=False)), element_required=False),
4289+
required=False,
4290+
),
4291+
)
4292+
arrow_schema = pa.schema([pa.field("col_list_with_struct", pa.list_(pa.struct([pa.field("a b", pa.int32())]), 2))])
4293+
values = pa.array(
4294+
[[{"a b": 1}, None], None, [{"a b": None}, {"a b": 2}], [{"a b": 3}, {"a b": 4}]], type=arrow_schema.field(0).type
4295+
)
4296+
batch = pa.RecordBatch.from_arrays([values], schema=arrow_schema)
4297+
4298+
result = _to_requested_schema(requested_schema, file_schema, batch)
4299+
assert result.column(0).to_pylist() == [[{"a_b": 1}, None], None, [{"a_b": None}, {"a_b": 2}], [{"a_b": 3}, {"a_b": 4}]]
4300+
4301+
# A sliced batch has to keep the nulls as well
4302+
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 3))
4303+
assert result.column(0).to_pylist() == [None, [{"a_b": None}, {"a_b": 2}], [{"a_b": 3}, {"a_b": 4}]]
4304+
4305+
4306+
def test__to_requested_schema_list_of_lists_of_renamed_structs() -> None:
4307+
# The renamed struct fields are only in the projected values, so every list around them has to be rebuilt
4308+
file_struct = StructType(NestedField(4, "a b", IntegerType(), required=False))
4309+
requested_struct = StructType(NestedField(4, "a_b", IntegerType(), required=False))
4310+
file_schema = Schema(
4311+
NestedField(
4312+
1,
4313+
"col",
4314+
ListType(2, ListType(3, file_struct, element_required=False), element_required=False),
4315+
required=False,
4316+
),
4317+
)
4318+
requested_schema = Schema(
4319+
NestedField(
4320+
1,
4321+
"col",
4322+
ListType(2, ListType(3, requested_struct, element_required=False), element_required=False),
4323+
required=False,
4324+
),
4325+
)
4326+
struct_type = pa.struct([pa.field("a b", pa.int32())])
4327+
builders: dict[str, Callable[[pa.DataType], pa.DataType]] = {
4328+
"list": pa.list_,
4329+
"large_list": pa.large_list,
4330+
"fixed_size_list": lambda value_type: pa.list_(value_type, 2),
4331+
}
4332+
rows = [[[{"a b": 1}, None], [{"a b": 2}, {"a b": None}]], None, [None, [{"a b": 3}, {"a b": 4}]]]
4333+
expected = [[[{"a_b": 1}, None], [{"a_b": 2}, {"a_b": None}]], None, [None, [{"a_b": 3}, {"a_b": 4}]]]
4334+
for outer_builder in builders.values():
4335+
for inner_builder in builders.values():
4336+
arrow_type = outer_builder(inner_builder(struct_type))
4337+
batch = pa.RecordBatch.from_arrays([pa.array(rows, type=arrow_type)], names=["col"])
4338+
4339+
result = _to_requested_schema(requested_schema, file_schema, batch)
4340+
assert result.column(0).to_pylist() == expected
4341+
4342+
# A sliced batch has to keep the nulls as well, also when the slice is empty
4343+
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 2))
4344+
assert result.column(0).to_pylist() == expected[1:]
4345+
assert _to_requested_schema(requested_schema, file_schema, batch.slice(1, 0)).num_rows == 0
4346+
4347+
# Control: a fixed-size list of integers goes through the same projection without any struct
4348+
ints_schema = Schema(NestedField(1, "col", ListType(2, IntegerType(), element_required=False), required=False))
4349+
ints = pa.RecordBatch.from_arrays([pa.array([[1, 2], None, [None, 4]], pa.list_(pa.int32(), 2))], names=["col"])
4350+
assert _to_requested_schema(ints_schema, ints_schema, ints).column(0).to_pylist() == [[1, 2], None, [None, 4]]
4351+
4352+
4353+
def test_append_fixed_size_list_of_structs(catalog: InMemoryCatalog) -> None:
4354+
arrow_table = pa.table(
4355+
{
4356+
"id": pa.array([1, 2], pa.int64()),
4357+
"v": pa.array(
4358+
[[{"a b": 1, "c": "x"}, {"a b": 2, "c": "y"}], [{"a b": None, "c": "z"}, None]],
4359+
pa.list_(pa.struct([("a b", pa.int32()), ("c", pa.string())]), 2),
4360+
),
4361+
}
4362+
)
4363+
catalog.create_namespace("default")
4364+
table = catalog.create_table("default.test_append_fixed_size_list_of_structs", schema=arrow_table.schema)
4365+
4366+
table.append(arrow_table)
4367+
4368+
assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()
4369+
4370+
4371+
def test__to_requested_schema_map_of_lists_of_renamed_structs() -> None:
4372+
# The renamed struct fields are in the list values of a map, so the map has to keep the projected values
4373+
file_schema = Schema(
4374+
NestedField(
4375+
1,
4376+
"col",
4377+
MapType(
4378+
2,
4379+
StringType(),
4380+
3,
4381+
ListType(4, StructType(NestedField(5, "a b", IntegerType(), required=False)), element_required=False),
4382+
value_required=False,
4383+
),
4384+
required=False,
4385+
),
4386+
)
4387+
requested_schema = Schema(
4388+
NestedField(
4389+
1,
4390+
"col",
4391+
MapType(
4392+
2,
4393+
StringType(),
4394+
3,
4395+
ListType(4, StructType(NestedField(5, "a_b", IntegerType(), required=False)), element_required=False),
4396+
value_required=False,
4397+
),
4398+
required=False,
4399+
),
4400+
)
4401+
struct_type = pa.struct([pa.field("a b", pa.int32())])
4402+
builders: list[Callable[[pa.DataType], pa.DataType]] = [pa.list_, pa.large_list, lambda value_type: pa.list_(value_type, 2)]
4403+
rows = [[("k", [{"a b": 1}, None])], None, [("j", None), ("m", [{"a b": 2}, {"a b": 3}])]]
4404+
expected = [[("k", [{"a_b": 1}, None])], None, [("j", None), ("m", [{"a_b": 2}, {"a_b": 3}])]]
4405+
for builder in builders:
4406+
batch = pa.RecordBatch.from_arrays([pa.array(rows, type=pa.map_(pa.string(), builder(struct_type)))], names=["col"])
4407+
4408+
result = _to_requested_schema(requested_schema, file_schema, batch)
4409+
assert result.column(0).to_pylist() == expected
4410+
4411+
# A sliced batch has to keep the null maps and the null list values as well
4412+
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 2))
4413+
assert result.column(0).to_pylist() == expected[1:]
4414+
4415+
4416+
def test_append_map_of_fixed_size_list_of_structs(catalog: InMemoryCatalog) -> None:
4417+
arrow_table = pa.table(
4418+
{
4419+
"v": pa.array(
4420+
[[("k", [{"a b": 1}, {"a b": 2}])]],
4421+
pa.map_(pa.string(), pa.list_(pa.struct([("a b", pa.int32())]), 2)),
4422+
),
4423+
}
4424+
)
4425+
catalog.create_namespace("default")
4426+
table = catalog.create_table("default.test_append_map_of_fixed_size_list_of_structs", schema=arrow_table.schema)
4427+
4428+
table.append(arrow_table)
4429+
4430+
assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()
4431+
4432+
4433+
def test_append_map_with_struct_keys(catalog: InMemoryCatalog) -> None:
4434+
# A struct key without nulls must not get a validity bitmap, or Arrow aborts on the rebuilt map
4435+
# (https://github.com/apache/arrow/issues/51029)
4436+
rows = [[({"a": 1}, 2)], None, [], [({"a": None}, 3)]]
4437+
arrow_table = pa.table({"v": pa.array(rows, pa.map_(pa.struct([("a", pa.int32())]), pa.int32()))})
4438+
key_type = StructType(NestedField(4, "a", IntegerType(), required=False))
4439+
schema = Schema(NestedField(1, "v", MapType(2, key_type, 3, IntegerType(), value_required=False), required=False))
4440+
batch = arrow_table.to_batches()[0]
4441+
for projected in (batch, batch.slice(1, 3)):
4442+
assert _to_requested_schema(schema, schema, projected).column(0).to_pylist() == projected.column(0).to_pylist()
4443+
4444+
catalog.create_namespace("default")
4445+
table = catalog.create_table("default.test_append_map_with_struct_keys", schema=arrow_table.schema)
4446+
4447+
table.append(arrow_table)
4448+
4449+
assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()
4450+
4451+
42744452
@skip_if_pyarrow_too_old
42754453
@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"])
42764454
def test_adls_account_name_from_location(scheme: str) -> None:

0 commit comments

Comments
 (0)