Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 31 additions & 29 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -2241,7 +2241,8 @@ def struct(
return pa.StructArray.from_arrays(
arrays=field_arrays,
fields=pa.struct(fields),
mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) else None,
# Avoid unknown null counts for non-null map keys: https://github.com/apache/arrow/issues/51029
mask=struct_array.is_null() if isinstance(struct_array, pa.StructArray) and struct_array.null_count else None,
)

def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array | None) -> pa.Array | None:
Expand All @@ -2250,18 +2251,23 @@ def field(self, field: NestedField, _: pa.Array | None, field_array: pa.Array |
def list(self, list_type: ListType, list_array: pa.Array | None, value_array: pa.Array | None) -> pa.Array | None:
if isinstance(list_array, (pa.ListArray, pa.LargeListArray, pa.FixedSizeListArray)) and value_array is not None:
list_initializer = pa.large_list if isinstance(list_array, pa.LargeListArray) else pa.list_
if isinstance(value_array, pa.StructArray):
# This can be removed once this has been fixed:
# https://github.com/apache/arrow/issues/38809
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists
list_array = pa.Array.from_buffers(
list_initializer(value_array.type),
len(list_array),
list_array.buffers()[:2],
list_array.null_count,
list_array.offset,
[value_array],
)
# Rebuild the list around the projected values, so the cast below does not cast the unprojected children.
# Keep the validity and offsets buffers of the original array, otherwise null lists become empty lists.
# A fixed-size list has no offsets buffer, so keep its validity buffer and list size instead.
if isinstance(list_array, pa.FixedSizeListArray):
arrow_list_type = pa.list_(value_array.type, list_array.type.list_size)
buffers = list_array.buffers()[:1]
else:
arrow_list_type = list_initializer(value_array.type)
buffers = list_array.buffers()[:2]
list_array = pa.Array.from_buffers(
arrow_list_type,
len(list_array),
buffers,
list_array.null_count,
list_array.offset,
[value_array],
)
value_array = self._cast_if_needed(list_type.element_field, value_array)
arrow_field = list_initializer(self._construct_field(list_type.element_field, value_array.type))
return list_array.cast(arrow_field)
Expand All @@ -2278,22 +2284,18 @@ def map(
self._construct_field(map_type.key_field, key_result.type),
self._construct_field(map_type.value_field, value_result.type),
)
if isinstance(value_result, pa.StructArray):
# Arrow does not allow reordering of fields, therefore we have to copy the array :(
entries = pa.StructArray.from_arrays(
[key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field]
)
# Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps
return pa.Array.from_buffers(
arrow_field,
len(map_array),
map_array.buffers()[:2],
map_array.null_count,
map_array.offset,
[entries],
)
else:
return map_array.cast(arrow_field)
entries = pa.StructArray.from_arrays(
[key_result, value_result], fields=[arrow_field.key_field, arrow_field.item_field]
)
# Keep the validity and offsets buffers of the original array, otherwise null maps become empty maps
return pa.Array.from_buffers(
arrow_field,
len(map_array),
map_array.buffers()[:2],
map_array.null_count,
map_array.offset,
[entries],
)
else:
return None

Expand Down
178 changes: 178 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -4271,6 +4271,184 @@ def test__to_requested_schema_null_list_of_structs(list_type: Callable[[pa.DataT
assert result.column(1).to_pylist() == [[], None, [2, None]]


def test__to_requested_schema_fixed_size_list_of_structs() -> None:
# The requested schema renames the struct field, like sanitize_column_names does on write
file_schema = Schema(
NestedField(
1,
"col_list_with_struct",
ListType(11, StructType(NestedField(111, "a b", IntegerType(), required=False)), element_required=False),
required=False,
),
)
requested_schema = Schema(
NestedField(
1,
"col_list_with_struct",
ListType(11, StructType(NestedField(111, "a_b", IntegerType(), required=False)), element_required=False),
required=False,
),
)
arrow_schema = pa.schema([pa.field("col_list_with_struct", pa.list_(pa.struct([pa.field("a b", pa.int32())]), 2))])
values = pa.array(
[[{"a b": 1}, None], None, [{"a b": None}, {"a b": 2}], [{"a b": 3}, {"a b": 4}]], type=arrow_schema.field(0).type
)
batch = pa.RecordBatch.from_arrays([values], schema=arrow_schema)

result = _to_requested_schema(requested_schema, file_schema, batch)
assert result.column(0).to_pylist() == [[{"a_b": 1}, None], None, [{"a_b": None}, {"a_b": 2}], [{"a_b": 3}, {"a_b": 4}]]

# A sliced batch has to keep the nulls as well
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 3))
assert result.column(0).to_pylist() == [None, [{"a_b": None}, {"a_b": 2}], [{"a_b": 3}, {"a_b": 4}]]


def test__to_requested_schema_list_of_lists_of_renamed_structs() -> None:
# The renamed struct fields are only in the projected values, so every list around them has to be rebuilt
file_struct = StructType(NestedField(4, "a b", IntegerType(), required=False))
requested_struct = StructType(NestedField(4, "a_b", IntegerType(), required=False))
file_schema = Schema(
NestedField(
1,
"col",
ListType(2, ListType(3, file_struct, element_required=False), element_required=False),
required=False,
),
)
requested_schema = Schema(
NestedField(
1,
"col",
ListType(2, ListType(3, requested_struct, element_required=False), element_required=False),
required=False,
),
)
struct_type = pa.struct([pa.field("a b", pa.int32())])
builders: dict[str, Callable[[pa.DataType], pa.DataType]] = {
"list": pa.list_,
"large_list": pa.large_list,
"fixed_size_list": lambda value_type: pa.list_(value_type, 2),
}
rows = [[[{"a b": 1}, None], [{"a b": 2}, {"a b": None}]], None, [None, [{"a b": 3}, {"a b": 4}]]]
expected = [[[{"a_b": 1}, None], [{"a_b": 2}, {"a_b": None}]], None, [None, [{"a_b": 3}, {"a_b": 4}]]]
for outer_builder in builders.values():
for inner_builder in builders.values():
arrow_type = outer_builder(inner_builder(struct_type))
batch = pa.RecordBatch.from_arrays([pa.array(rows, type=arrow_type)], names=["col"])

result = _to_requested_schema(requested_schema, file_schema, batch)
assert result.column(0).to_pylist() == expected

# A sliced batch has to keep the nulls as well, also when the slice is empty
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 2))
assert result.column(0).to_pylist() == expected[1:]
assert _to_requested_schema(requested_schema, file_schema, batch.slice(1, 0)).num_rows == 0

# Control: a fixed-size list of integers goes through the same projection without any struct
ints_schema = Schema(NestedField(1, "col", ListType(2, IntegerType(), element_required=False), required=False))
ints = pa.RecordBatch.from_arrays([pa.array([[1, 2], None, [None, 4]], pa.list_(pa.int32(), 2))], names=["col"])
assert _to_requested_schema(ints_schema, ints_schema, ints).column(0).to_pylist() == [[1, 2], None, [None, 4]]


def test_append_fixed_size_list_of_structs(catalog: InMemoryCatalog) -> None:
arrow_table = pa.table(
{
"id": pa.array([1, 2], pa.int64()),
"v": pa.array(
[[{"a b": 1, "c": "x"}, {"a b": 2, "c": "y"}], [{"a b": None, "c": "z"}, None]],
pa.list_(pa.struct([("a b", pa.int32()), ("c", pa.string())]), 2),
),
}
)
catalog.create_namespace("default")
table = catalog.create_table("default.test_append_fixed_size_list_of_structs", schema=arrow_table.schema)

table.append(arrow_table)

assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()


def test__to_requested_schema_map_of_lists_of_renamed_structs() -> None:
# The renamed struct fields are in the list values of a map, so the map has to keep the projected values
file_schema = Schema(
NestedField(
1,
"col",
MapType(
2,
StringType(),
3,
ListType(4, StructType(NestedField(5, "a b", IntegerType(), required=False)), element_required=False),
value_required=False,
),
required=False,
),
)
requested_schema = Schema(
NestedField(
1,
"col",
MapType(
2,
StringType(),
3,
ListType(4, StructType(NestedField(5, "a_b", IntegerType(), required=False)), element_required=False),
value_required=False,
),
required=False,
),
)
struct_type = pa.struct([pa.field("a b", pa.int32())])
builders: list[Callable[[pa.DataType], pa.DataType]] = [pa.list_, pa.large_list, lambda value_type: pa.list_(value_type, 2)]
rows = [[("k", [{"a b": 1}, None])], None, [("j", None), ("m", [{"a b": 2}, {"a b": 3}])]]
expected = [[("k", [{"a_b": 1}, None])], None, [("j", None), ("m", [{"a_b": 2}, {"a_b": 3}])]]
for builder in builders:
batch = pa.RecordBatch.from_arrays([pa.array(rows, type=pa.map_(pa.string(), builder(struct_type)))], names=["col"])

result = _to_requested_schema(requested_schema, file_schema, batch)
assert result.column(0).to_pylist() == expected

# A sliced batch has to keep the null maps and the null list values as well
result = _to_requested_schema(requested_schema, file_schema, batch.slice(1, 2))
assert result.column(0).to_pylist() == expected[1:]


def test_append_map_of_fixed_size_list_of_structs(catalog: InMemoryCatalog) -> None:
arrow_table = pa.table(
{
"v": pa.array(
[[("k", [{"a b": 1}, {"a b": 2}])]],
pa.map_(pa.string(), pa.list_(pa.struct([("a b", pa.int32())]), 2)),
),
}
)
catalog.create_namespace("default")
table = catalog.create_table("default.test_append_map_of_fixed_size_list_of_structs", schema=arrow_table.schema)

table.append(arrow_table)

assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()


def test_append_map_with_struct_keys(catalog: InMemoryCatalog) -> None:
# A struct key without nulls must not get a validity bitmap, or Arrow aborts on the rebuilt map
# (https://github.com/apache/arrow/issues/51029)
rows = [[({"a": 1}, 2)], None, [], [({"a": None}, 3)]]
arrow_table = pa.table({"v": pa.array(rows, pa.map_(pa.struct([("a", pa.int32())]), pa.int32()))})
key_type = StructType(NestedField(4, "a", IntegerType(), required=False))
schema = Schema(NestedField(1, "v", MapType(2, key_type, 3, IntegerType(), value_required=False), required=False))
batch = arrow_table.to_batches()[0]
for projected in (batch, batch.slice(1, 3)):
assert _to_requested_schema(schema, schema, projected).column(0).to_pylist() == projected.column(0).to_pylist()

catalog.create_namespace("default")
table = catalog.create_table("default.test_append_map_with_struct_keys", schema=arrow_table.schema)

table.append(arrow_table)

assert table.scan().to_arrow().to_pylist() == arrow_table.to_pylist()


@skip_if_pyarrow_too_old
@pytest.mark.parametrize("scheme", ["abfs", "abfss", "wasb", "wasbs"])
def test_adls_account_name_from_location(scheme: str) -> None:
Expand Down
Loading