diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a15fe19e71..27114ef33c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1265,7 +1265,26 @@ def pyarrow_to_schema( name_mapping: NameMapping | None = None, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, + prune_fields_without_ids: bool = False, ) -> Schema: + """Convert a PyArrow schema to an Iceberg schema. + + Args: + schema: The PyArrow schema to convert. + name_mapping: Used when the schema has no field IDs. + downcast_ns_timestamp_to_us: Downcast nanosecond timestamps to microseconds. + format_version: The Iceberg table format version. + prune_fields_without_ids: If some fields have field IDs, drop the ones that do not + (including lists and maps with an element, key, or value without one) instead of + falling back to the name mapping, like the Java Parquet reader does. Only meant for + reading data files. Writes keep the stricter check, so extra columns are rejected + instead of silently dropped. + """ + if prune_fields_without_ids: + # Prune before conversion, so an unsupported type in a column without an ID does not fail the read + pruned_schema = _prune_fields_without_ids(schema) + if len(pruned_schema) > 0: + schema = pruned_schema has_ids = visit_pyarrow(schema, _HasIds()) if has_ids: return visit_pyarrow( @@ -1282,6 +1301,50 @@ def pyarrow_to_schema( ) +def _prune_fields_without_ids(schema: pa.Schema) -> pa.Schema: + """Drop fields without a field ID, so only the columns with IDs are read. + + A list is dropped when its element has no ID, and a map when its key or value has none, + since neither can be represented in Iceberg without them. A struct whose children all + get dropped is dropped as well. + """ + fields = [pruned for field in schema if (pruned := _prune_field_without_id(field)) is not None] + return pa.schema(fields, metadata=schema.metadata) + + +def _prune_field_without_id(field: pa.Field) -> pa.Field | None: + if _get_field_id(field) is None: + return None + pruned_type = _prune_type_without_ids(field.type) + if pruned_type is None: + return None + return field.with_type(pruned_type) + + +def _prune_type_without_ids(arrow_type: pa.DataType) -> pa.DataType | None: + if pa.types.is_struct(arrow_type): + children = [arrow_type.field(i) for i in range(arrow_type.num_fields)] + pruned_children = [pruned for child in children if (pruned := _prune_field_without_id(child)) is not None] + return pa.struct(pruned_children) if pruned_children else None + elif isinstance(arrow_type, (pa.ListType, pa.LargeListType, pa.FixedSizeListType)): + element_field = _prune_field_without_id(arrow_type.value_field) + if element_field is None: + return None + if isinstance(arrow_type, pa.LargeListType): + return pa.large_list(element_field) + if isinstance(arrow_type, pa.FixedSizeListType): + return pa.list_(element_field, arrow_type.list_size) + return pa.list_(element_field) + elif isinstance(arrow_type, pa.MapType): + key_field = _prune_field_without_id(arrow_type.key_field) + value_field = _prune_field_without_id(arrow_type.item_field) + if key_field is None or value_field is None: + # Keeping a map without its value (or key) would change its type, so drop the whole map + return None + return pa.map_(key_field, value_field, keys_sorted=arrow_type.keys_sorted) + return arrow_type + + def _pyarrow_to_schema_without_ids( schema: pa.Schema, downcast_ns_timestamp_to_us: bool = False, @@ -1779,7 +1842,11 @@ def _task_to_record_batches( downcast_ns_timestamp_to_us if downcast_ns_timestamp_to_us is not None else format_version <= 2 ) file_schema = pyarrow_to_schema( - physical_schema, name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version + physical_schema, + name_mapping, + downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, + format_version=format_version, + prune_fields_without_ids=True, ) # Apply column projection rules: https://iceberg.apache.org/spec/#column-projection diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..b73b0889f8 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3342,6 +3342,41 @@ def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: _check_pyarrow_schema_compatible(table_schema_simple, other_schema) +def test_schema_mismatch_additional_field_without_id(table_schema_simple: Schema) -> None: + # Appending a table read from a data file (with field IDs) plus a new column without one + # must still be rejected, rather than dropping the new column + other_schema = pa.schema( + ( + pa.field("foo", pa.string(), nullable=True, metadata={"PARQUET:field_id": "1"}), + pa.field("bar", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "2"}), + pa.field("baz", pa.bool_(), nullable=True, metadata={"PARQUET:field_id": "3"}), + pa.field("country", pa.string(), nullable=True), + ) + ) + + with pytest.raises( + ValueError, match=r"PyArrow table contains more columns: country. Update the schema first \(hint, use union_by_name\)." + ): + _check_pyarrow_schema_compatible(table_schema_simple, other_schema) + + +def test_projection_partial_field_ids(tmpdir: str) -> None: + # Columns without a field ID are ignored on read, even if their type is not supported + schema = Schema(NestedField(1, "foo", StringType(), required=False)) + arrow_schema = pa.schema( + [ + pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}), + pa.field("_row-id-col-922d6355", pa.time32("ms")), + ] + ) + table = pa.Table.from_arrays([pa.array(["a", "b", "c"]), pa.array([1, 2, 3], type=pa.time32("ms"))], schema=arrow_schema) + file = _write_table_to_file(f"{tmpdir}/partial-ids.parquet", arrow_schema, table) + + result = project(schema, [file]) + assert result.column_names == ["foo"] + assert result["foo"].to_pylist() == ["a", "b", "c"] + + def test_schema_compatible(table_schema_simple: Schema) -> None: try: _check_pyarrow_schema_compatible(table_schema_simple, table_schema_simple.as_arrow()) diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..f176ae6a99 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -460,6 +460,152 @@ def test_nested_schema_has_partial_missing_ids() -> None: assert not has_ids +def _id(field_id: int) -> dict[str, str]: + return {"PARQUET:field_id": str(field_id)} + + +def test_pyarrow_to_schema_prunes_fields_without_ids() -> None: + # A Databricks UniForm file may attach IDs only to data columns, leaving + # row-tracking columns unannotated. Without pruning, the file falls back to + # name mapping, which fails when physical names differ from the mapped names. + file_schema = pa.schema( + [ + pa.field("col-2b93b371", pa.string(), metadata=_id(1)), + pa.field("_row-id-col-922d6355", pa.int64()), + ] + ) + expected = Schema(NestedField(1, "col-2b93b371", StringType(), required=False)) + name_mappings = [None, NameMapping([MappedField(field_id=1, names=["store_id"])])] + for name_mapping in name_mappings: + assert pyarrow_to_schema(file_schema, name_mapping, prune_fields_without_ids=True) == expected + + +def test_pyarrow_to_schema_partial_ids_without_pruning() -> None: + # Without pruning (e.g. on write), partial IDs still fall back to the name mapping + file_schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("bar", pa.int64()), + ] + ) + with pytest.raises(ValueError, match="Parquet file does not have field-ids"): + pyarrow_to_schema(file_schema) + + +def test_pyarrow_to_schema_prunes_before_type_conversion() -> None: + # time32[ms] is not supported, but the column has no ID so it is never read + file_schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("bar", pa.time32("ms")), + ] + ) + expected = Schema(NestedField(1, "foo", StringType(), required=False)) + assert pyarrow_to_schema(file_schema, prune_fields_without_ids=True) == expected + + +def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), + pa.field( + "quux", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata=_id(7)), + pa.field( + "value", + pa.map_(pa.field("key", pa.string(), nullable=False), pa.field("value", pa.int32())), + nullable=False, + ), + ), + nullable=False, + metadata={"PARQUET:field_id": "6", "doc": "quux doc"}, + ), + pa.field( + "person", + pa.struct( + [ + pa.field("name", pa.string(), metadata=_id(16)), + pa.field("age", pa.int32()), + ] + ), + metadata=_id(15), + ), + ] + ) + actual = str(pyarrow_to_schema(schema, prune_fields_without_ids=True)) + expected = """table { + 1: foo: required string (foo doc) + 15: person: optional struct<16: name: optional string> +}""" + assert actual == expected + + +def test_pyarrow_to_schema_prunes_list_without_element_id() -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("no_element_id", pa.list_(pa.field("element", pa.int32())), metadata=_id(2)), + pa.field("with_element_id", pa.list_(pa.field("element", pa.int32(), metadata=_id(4))), metadata=_id(3)), + ] + ) + expected = Schema( + NestedField(1, "foo", StringType(), required=False), + NestedField(3, "with_element_id", ListType(4, IntegerType(), element_required=False), required=False), + ) + assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected + + +@pytest.mark.parametrize( + "key_metadata, value_metadata", + [ + (None, _id(4)), # value only + (_id(3), None), # key only, the value can't be kept on its own so the whole map goes + ], +) +def test_pyarrow_to_schema_prunes_map_without_key_or_value_id( + key_metadata: dict[str, str] | None, value_metadata: dict[str, str] | None +) -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field( + "partial_map", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata=key_metadata), + pa.field("value", pa.int32(), metadata=value_metadata), + ), + metadata=_id(2), + ), + pa.field( + "full_map", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata=_id(6)), + pa.field("value", pa.int32(), metadata=_id(7)), + ), + metadata=_id(5), + ), + ] + ) + expected = Schema( + NestedField(1, "foo", StringType(), required=False), + NestedField( + 5, + "full_map", + MapType(key_id=6, key_type=StringType(), value_id=7, value_type=IntegerType(), value_required=False), + required=False, + ), + ) + assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected + + +def test_pyarrow_to_schema_without_any_ids_uses_name_mapping_when_pruning() -> None: + schema = pa.schema([pa.field("foo", pa.string())]) + name_mapping = NameMapping([MappedField(field_id=1, names=["foo"])]) + expected = Schema(NestedField(1, "foo", StringType(), required=False)) + assert pyarrow_to_schema(schema, name_mapping, prune_fields_without_ids=True) == expected + + def test_pyarrow_schema_to_schema_missing_ids_and_name_mapping(pyarrow_schema_simple_without_ids: pa.Schema) -> None: schema = pyarrow_schema_simple_without_ids with pytest.raises(ValueError) as exc_info: