Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 68 additions & 1 deletion pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1265,7 +1265,26 @@ def pyarrow_to_schema(
name_mapping: NameMapping | None = None,
downcast_ns_timestamp_to_us: bool = False,
format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION,
prune_fields_without_ids: bool = False,
) -> Schema:
"""Convert a PyArrow schema to an Iceberg schema.

Args:
schema: The PyArrow schema to convert.
name_mapping: Used when the schema has no field IDs.
downcast_ns_timestamp_to_us: Downcast nanosecond timestamps to microseconds.
format_version: The Iceberg table format version.
prune_fields_without_ids: If some fields have field IDs, drop the ones that do not
(including lists and maps with an element, key, or value without one) instead of
falling back to the name mapping, like the Java Parquet reader does. Only meant for
reading data files. Writes keep the stricter check, so extra columns are rejected
instead of silently dropped.
"""
if prune_fields_without_ids:
# Prune before conversion, so an unsupported type in a column without an ID does not fail the read
pruned_schema = _prune_fields_without_ids(schema)
if len(pruned_schema) > 0:
schema = pruned_schema
has_ids = visit_pyarrow(schema, _HasIds())
if has_ids:
return visit_pyarrow(
Expand All @@ -1282,6 +1301,50 @@ def pyarrow_to_schema(
)


def _prune_fields_without_ids(schema: pa.Schema) -> pa.Schema:
"""Drop fields without a field ID, so only the columns with IDs are read.

A list is dropped when its element has no ID, and a map when its key or value has none,
since neither can be represented in Iceberg without them. A struct whose children all
get dropped is dropped as well.
"""
fields = [pruned for field in schema if (pruned := _prune_field_without_id(field)) is not None]
return pa.schema(fields, metadata=schema.metadata)


def _prune_field_without_id(field: pa.Field) -> pa.Field | None:
if _get_field_id(field) is None:
return None
pruned_type = _prune_type_without_ids(field.type)
if pruned_type is None:
return None
return field.with_type(pruned_type)


def _prune_type_without_ids(arrow_type: pa.DataType) -> pa.DataType | None:
if pa.types.is_struct(arrow_type):
children = [arrow_type.field(i) for i in range(arrow_type.num_fields)]
pruned_children = [pruned for child in children if (pruned := _prune_field_without_id(child)) is not None]
return pa.struct(pruned_children) if pruned_children else None
elif isinstance(arrow_type, (pa.ListType, pa.LargeListType, pa.FixedSizeListType)):
element_field = _prune_field_without_id(arrow_type.value_field)
if element_field is None:
return None
if isinstance(arrow_type, pa.LargeListType):
return pa.large_list(element_field)
if isinstance(arrow_type, pa.FixedSizeListType):
return pa.list_(element_field, arrow_type.list_size)
return pa.list_(element_field)
elif isinstance(arrow_type, pa.MapType):
key_field = _prune_field_without_id(arrow_type.key_field)
value_field = _prune_field_without_id(arrow_type.item_field)
if key_field is None or value_field is None:
# Keeping a map without its value (or key) would change its type, so drop the whole map
return None
return pa.map_(key_field, value_field, keys_sorted=arrow_type.keys_sorted)
return arrow_type


def _pyarrow_to_schema_without_ids(
schema: pa.Schema,
downcast_ns_timestamp_to_us: bool = False,
Expand Down Expand Up @@ -1779,7 +1842,11 @@ def _task_to_record_batches(
downcast_ns_timestamp_to_us if downcast_ns_timestamp_to_us is not None else format_version <= 2
)
file_schema = pyarrow_to_schema(
physical_schema, name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version
physical_schema,
name_mapping,
downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us,
format_version=format_version,
prune_fields_without_ids=True,
)

# Apply column projection rules: https://iceberg.apache.org/spec/#column-projection
Expand Down
35 changes: 35 additions & 0 deletions tests/io/test_pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -3342,6 +3342,41 @@ def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None:
_check_pyarrow_schema_compatible(table_schema_simple, other_schema)


def test_schema_mismatch_additional_field_without_id(table_schema_simple: Schema) -> None:
# Appending a table read from a data file (with field IDs) plus a new column without one
# must still be rejected, rather than dropping the new column
other_schema = pa.schema(
(
pa.field("foo", pa.string(), nullable=True, metadata={"PARQUET:field_id": "1"}),
pa.field("bar", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "2"}),
pa.field("baz", pa.bool_(), nullable=True, metadata={"PARQUET:field_id": "3"}),
pa.field("country", pa.string(), nullable=True),
)
)

with pytest.raises(
ValueError, match=r"PyArrow table contains more columns: country. Update the schema first \(hint, use union_by_name\)."
):
_check_pyarrow_schema_compatible(table_schema_simple, other_schema)


def test_projection_partial_field_ids(tmpdir: str) -> None:
# Columns without a field ID are ignored on read, even if their type is not supported
schema = Schema(NestedField(1, "foo", StringType(), required=False))
arrow_schema = pa.schema(
[
pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}),
pa.field("_row-id-col-922d6355", pa.time32("ms")),
]
)
table = pa.Table.from_arrays([pa.array(["a", "b", "c"]), pa.array([1, 2, 3], type=pa.time32("ms"))], schema=arrow_schema)
file = _write_table_to_file(f"{tmpdir}/partial-ids.parquet", arrow_schema, table)

result = project(schema, [file])
assert result.column_names == ["foo"]
assert result["foo"].to_pylist() == ["a", "b", "c"]


def test_schema_compatible(table_schema_simple: Schema) -> None:
try:
_check_pyarrow_schema_compatible(table_schema_simple, table_schema_simple.as_arrow())
Expand Down
146 changes: 146 additions & 0 deletions tests/io/test_pyarrow_visitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,152 @@ def test_nested_schema_has_partial_missing_ids() -> None:
assert not has_ids


def _id(field_id: int) -> dict[str, str]:
return {"PARQUET:field_id": str(field_id)}


def test_pyarrow_to_schema_prunes_fields_without_ids() -> None:
# A Databricks UniForm file may attach IDs only to data columns, leaving
# row-tracking columns unannotated. Without pruning, the file falls back to
# name mapping, which fails when physical names differ from the mapped names.
file_schema = pa.schema(
[
pa.field("col-2b93b371", pa.string(), metadata=_id(1)),
pa.field("_row-id-col-922d6355", pa.int64()),
]
)
expected = Schema(NestedField(1, "col-2b93b371", StringType(), required=False))
name_mappings = [None, NameMapping([MappedField(field_id=1, names=["store_id"])])]
for name_mapping in name_mappings:
assert pyarrow_to_schema(file_schema, name_mapping, prune_fields_without_ids=True) == expected


def test_pyarrow_to_schema_partial_ids_without_pruning() -> None:
# Without pruning (e.g. on write), partial IDs still fall back to the name mapping
file_schema = pa.schema(
[
pa.field("foo", pa.string(), metadata=_id(1)),
pa.field("bar", pa.int64()),
]
)
with pytest.raises(ValueError, match="Parquet file does not have field-ids"):
pyarrow_to_schema(file_schema)


def test_pyarrow_to_schema_prunes_before_type_conversion() -> None:
# time32[ms] is not supported, but the column has no ID so it is never read
file_schema = pa.schema(
[
pa.field("foo", pa.string(), metadata=_id(1)),
pa.field("bar", pa.time32("ms")),
]
)
expected = Schema(NestedField(1, "foo", StringType(), required=False))
assert pyarrow_to_schema(file_schema, prune_fields_without_ids=True) == expected


def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None:
schema = pa.schema(
[
pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}),
pa.field(
"quux",
pa.map_(
pa.field("key", pa.string(), nullable=False, metadata=_id(7)),
pa.field(
"value",
pa.map_(pa.field("key", pa.string(), nullable=False), pa.field("value", pa.int32())),
nullable=False,
),
),
nullable=False,
metadata={"PARQUET:field_id": "6", "doc": "quux doc"},
),
pa.field(
"person",
pa.struct(
[
pa.field("name", pa.string(), metadata=_id(16)),
pa.field("age", pa.int32()),
]
),
metadata=_id(15),
),
]
)
actual = str(pyarrow_to_schema(schema, prune_fields_without_ids=True))
expected = """table {
1: foo: required string (foo doc)
15: person: optional struct<16: name: optional string>
}"""
assert actual == expected


def test_pyarrow_to_schema_prunes_list_without_element_id() -> None:
schema = pa.schema(
[
pa.field("foo", pa.string(), metadata=_id(1)),
pa.field("no_element_id", pa.list_(pa.field("element", pa.int32())), metadata=_id(2)),
pa.field("with_element_id", pa.list_(pa.field("element", pa.int32(), metadata=_id(4))), metadata=_id(3)),
]
)
expected = Schema(
NestedField(1, "foo", StringType(), required=False),
NestedField(3, "with_element_id", ListType(4, IntegerType(), element_required=False), required=False),
)
assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected


@pytest.mark.parametrize(
"key_metadata, value_metadata",
[
(None, _id(4)), # value only
(_id(3), None), # key only, the value can't be kept on its own so the whole map goes
],
)
def test_pyarrow_to_schema_prunes_map_without_key_or_value_id(
key_metadata: dict[str, str] | None, value_metadata: dict[str, str] | None
) -> None:
schema = pa.schema(
[
pa.field("foo", pa.string(), metadata=_id(1)),
pa.field(
"partial_map",
pa.map_(
pa.field("key", pa.string(), nullable=False, metadata=key_metadata),
pa.field("value", pa.int32(), metadata=value_metadata),
),
metadata=_id(2),
),
pa.field(
"full_map",
pa.map_(
pa.field("key", pa.string(), nullable=False, metadata=_id(6)),
pa.field("value", pa.int32(), metadata=_id(7)),
),
metadata=_id(5),
),
]
)
expected = Schema(
NestedField(1, "foo", StringType(), required=False),
NestedField(
5,
"full_map",
MapType(key_id=6, key_type=StringType(), value_id=7, value_type=IntegerType(), value_required=False),
required=False,
),
)
assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected


def test_pyarrow_to_schema_without_any_ids_uses_name_mapping_when_pruning() -> None:
schema = pa.schema([pa.field("foo", pa.string())])
name_mapping = NameMapping([MappedField(field_id=1, names=["foo"])])
expected = Schema(NestedField(1, "foo", StringType(), required=False))
assert pyarrow_to_schema(schema, name_mapping, prune_fields_without_ids=True) == expected


def test_pyarrow_schema_to_schema_missing_ids_and_name_mapping(pyarrow_schema_simple_without_ids: pa.Schema) -> None:
schema = pyarrow_schema_simple_without_ids
with pytest.raises(ValueError) as exc_info:
Expand Down
Loading