From df5082ff17509fe78ddc030a3f6a26ba5e92ca58 Mon Sep 17 00:00:00 2001 From: linzch3 <20380442+linzch3@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:18:59 +0000 Subject: [PATCH 1/2] Use field IDs when any Parquet column has one _HasIds previously required every column to have an ID. A file with IDs on only some columns, such as a Databricks UniForm table with row-tracking columns, fell back to name mapping and failed when physical names differed from the mapped names. Match Java's ParquetSchemaUtil.hasIds (any) and prune fields without IDs, including incomplete nested types. --- pyiceberg/io/pyarrow.py | 58 ++++++++++++++++++------------ tests/io/test_pyarrow.py | 6 ++-- tests/io/test_pyarrow_visitor.py | 62 +++++++++++++++++++++++++++++--- 3 files changed, 98 insertions(+), 28 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a15fe19e71..a3eb6ca10d 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1268,9 +1268,11 @@ def pyarrow_to_schema( ) -> Schema: has_ids = visit_pyarrow(schema, _HasIds()) if has_ids: - return visit_pyarrow( + result = visit_pyarrow( schema, _ConvertToIceberg(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version) ) + assert isinstance(result, Schema) + return result elif name_mapping is not None: schema_without_ids = _pyarrow_to_schema_without_ids( schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version @@ -1287,10 +1289,12 @@ def _pyarrow_to_schema_without_ids( downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, ) -> Schema: - return visit_pyarrow( + result = visit_pyarrow( schema, _ConvertToIcebergWithoutIDs(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version), ) + assert isinstance(result, Schema) + return result def _pyarrow_schema_ensure_large_types(schema: pa.Schema) -> pa.Schema: @@ -1448,32 +1452,39 @@ def _get_field_id(field: pa.Field) -> int | None: class _HasIds(PyArrowSchemaVisitor[bool]): + """Return whether any field in the Arrow schema has an Iceberg field ID. + + Matches Java's ParquetSchemaUtil.hasIds, which is true if any column has an ID. + Fields without IDs are later pruned by _ConvertToIceberg. + """ + def schema(self, schema: pa.Schema, struct_result: bool) -> bool: return struct_result def struct(self, struct: pa.StructType, field_results: builtins.list[bool]) -> bool: - return all(field_results) + return any(field_results) def field(self, field: pa.Field, field_result: bool) -> bool: - return all([_get_field_id(field) is not None, field_result]) + return _get_field_id(field) is not None or field_result def list(self, list_type: pa.ListType, element_result: bool) -> bool: element_field = list_type.value_field element_id = _get_field_id(element_field) - return element_result and element_id is not None + return element_result or element_id is not None def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: key_field = map_type.key_field key_id = _get_field_id(key_field) value_field = map_type.item_field value_id = _get_field_id(value_field) - return all([key_id is not None, value_id is not None, key_result, value_result]) + return any([key_id is not None, value_id is not None, key_result, value_result]) def primitive(self, primitive: pa.DataType) -> bool: - return True + # IDs live on pa.Field metadata, not on primitive types + return False -class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema]): +class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema | None]): """Converts PyArrowSchema to Iceberg Schema.""" _field_names: builtins.list[str] @@ -1485,34 +1496,37 @@ def __init__( self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us self._format_version = format_version - def _field_id(self, field: pa.Field) -> int: - if (field_id := _get_field_id(field)) is not None: - return field_id - else: - raise ValueError(f"Cannot convert {field} to Iceberg Field as field_id is empty.") + def _field_id(self, field: pa.Field) -> int | None: + return _get_field_id(field) - def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: - return Schema(*struct_result.fields) + def schema(self, schema: pa.Schema, struct_result: StructType | None) -> Schema: + return Schema(*(struct_result.fields if struct_result is not None else ())) - def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: - return StructType(*field_results) + def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField | None]) -> StructType | None: + fields = tuple(field for field in field_results if field is not None) + return StructType(*fields) if fields else None - def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: + def field(self, field: pa.Field, field_result: IcebergType | None) -> NestedField | None: field_id = self._field_id(field) + if field_id is None or field_result is None: + return None field_doc = doc_str.decode() if (field.metadata and (doc_str := field.metadata.get(PYARROW_FIELD_DOC_KEY))) else None - field_type = field_result - return NestedField(field_id, field.name, field_type, required=not field.nullable, doc=field_doc) + return NestedField(field_id, field.name, field_result, required=not field.nullable, doc=field_doc) - def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: + def list(self, list_type: pa.ListType, element_result: IcebergType | None) -> ListType | None: element_field = list_type.value_field element_id = self._field_id(element_field) + if element_id is None or element_result is None: + return None return ListType(element_id, element_result, element_required=not element_field.nullable) - def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: + def map(self, map_type: pa.MapType, key_result: IcebergType | None, value_result: IcebergType | None) -> MapType | None: key_field = map_type.key_field key_id = self._field_id(key_field) value_field = map_type.item_field value_id = self._field_id(value_field) + if key_id is None or value_id is None or key_result is None or value_result is None: + return None return MapType(key_id, key_result, value_id, value_result, value_required=not value_field.nullable) def primitive(self, primitive: pa.DataType) -> PrimitiveType: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index a9c418e50a..489b3e765e 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3255,10 +3255,11 @@ def test_schema_compatible_missing_nullable_field_nested(table_schema_nested: Sc "person", pa.struct( [ - pa.field("age", pa.int32(), nullable=False), + pa.field("age", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "17"}), ] ), nullable=True, + metadata={"PARQUET:field_id": "15"}, ), ) try: @@ -3276,10 +3277,11 @@ def test_schema_mismatch_missing_required_field_nested(table_schema_nested: Sche "person", pa.struct( [ - pa.field("name", pa.string(), nullable=True), + pa.field("name", pa.string(), nullable=True, metadata={"PARQUET:field_id": "16"}), ] ), nullable=True, + metadata={"PARQUET:field_id": "15"}, ), ) expected = """Mismatch in fields: diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index e98d76e262..9d42634f8d 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -377,7 +377,7 @@ def test_simple_schema_has_missing_ids() -> None: assert not has_ids -def test_simple_schema_has_missing_ids_partial() -> None: +def test_simple_schema_has_partial_ids() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -386,7 +386,7 @@ def test_simple_schema_has_missing_ids_partial() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert not has_ids + assert has_ids def test_nested_schema_has_missing_ids() -> None: @@ -436,7 +436,7 @@ def test_nested_schema_has_ids() -> None: assert has_ids -def test_nested_schema_has_partial_missing_ids() -> None: +def test_nested_schema_has_partial_ids() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -457,7 +457,61 @@ def test_nested_schema_has_partial_missing_ids() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert not has_ids + assert has_ids + + +def test_pyarrow_to_schema_prunes_fields_without_ids() -> None: + # A Databricks UniForm file may attach IDs only to data columns, leaving + # row-tracking columns unannotated. Previously _HasIds required every + # column to have an ID and fell back to name mapping, which failed when + # physical names differed from the mapped names. + file_schema = pa.schema( + [ + pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}), + pa.field("_row-id-col-922d6355", pa.int64()), + ] + ) + expected = Schema(NestedField(1, "col-2b93b371", StringType(), required=False)) + name_mappings = [None, NameMapping([MappedField(field_id=1, names=["store_id"])])] + for name_mapping in name_mappings: + assert pyarrow_to_schema(file_schema, name_mapping) == expected + + +def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), + pa.field( + "quux", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata={"PARQUET:field_id": "7"}), + pa.field( + "value", + pa.map_(pa.field("key", pa.string(), nullable=False), pa.field("value", pa.int32())), + nullable=False, + ), + ), + nullable=False, + metadata={"PARQUET:field_id": "6", "doc": "quux doc"}, + ), + pa.field( + "person", + pa.struct( + [ + pa.field("name", pa.string(), metadata={"PARQUET:field_id": "16"}), + pa.field("age", pa.int32()), + ] + ), + metadata={"PARQUET:field_id": "15"}, + ), + ] + ) + actual = str(pyarrow_to_schema(schema)) + expected = """table { + 1: foo: required string (foo doc) + 15: person: optional struct<16: name: optional string> +}""" + assert actual == expected def test_pyarrow_schema_to_schema_missing_ids_and_name_mapping(pyarrow_schema_simple_without_ids: pa.Schema) -> None: From 6862dd6c1e171226b459245d2df9ac127b4d2423 Mon Sep 17 00:00:00 2001 From: linzch3 <20380442+linzch3@users.noreply.github.com> Date: Thu, 19 Nov 2026 03:05:52 +0000 Subject: [PATCH 2/2] Prune fields without IDs only when reading data files Review feedback: keep the old behavior on the write path, so an append with an extra column that has no field ID is still rejected instead of silently dropping that column. _HasIds and _ConvertToIceberg are back to their previous behavior. pyarrow_to_schema gets a prune_fields_without_ids flag, used only when reading data files. It drops fields without IDs from the Arrow schema before conversion, so an unsupported type in such a column (e.g. time32[ms]) does not fail the read. Lists without an element ID and maps without a key or value ID are dropped as a whole. Add tests for the list element / map key / map value ID guards, the pre-conversion pruning, a read of a file with partial IDs, and the append rejection. --- pyiceberg/io/pyarrow.py | 127 ++++++++++++++++++++++--------- tests/io/test_pyarrow.py | 41 +++++++++- tests/io/test_pyarrow_visitor.py | 118 ++++++++++++++++++++++++---- 3 files changed, 232 insertions(+), 54 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a3eb6ca10d..27114ef33c 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1265,14 +1265,31 @@ def pyarrow_to_schema( name_mapping: NameMapping | None = None, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, + prune_fields_without_ids: bool = False, ) -> Schema: + """Convert a PyArrow schema to an Iceberg schema. + + Args: + schema: The PyArrow schema to convert. + name_mapping: Used when the schema has no field IDs. + downcast_ns_timestamp_to_us: Downcast nanosecond timestamps to microseconds. + format_version: The Iceberg table format version. + prune_fields_without_ids: If some fields have field IDs, drop the ones that do not + (including lists and maps with an element, key, or value without one) instead of + falling back to the name mapping, like the Java Parquet reader does. Only meant for + reading data files. Writes keep the stricter check, so extra columns are rejected + instead of silently dropped. + """ + if prune_fields_without_ids: + # Prune before conversion, so an unsupported type in a column without an ID does not fail the read + pruned_schema = _prune_fields_without_ids(schema) + if len(pruned_schema) > 0: + schema = pruned_schema has_ids = visit_pyarrow(schema, _HasIds()) if has_ids: - result = visit_pyarrow( + return visit_pyarrow( schema, _ConvertToIceberg(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version) ) - assert isinstance(result, Schema) - return result elif name_mapping is not None: schema_without_ids = _pyarrow_to_schema_without_ids( schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version @@ -1284,17 +1301,59 @@ def pyarrow_to_schema( ) +def _prune_fields_without_ids(schema: pa.Schema) -> pa.Schema: + """Drop fields without a field ID, so only the columns with IDs are read. + + A list is dropped when its element has no ID, and a map when its key or value has none, + since neither can be represented in Iceberg without them. A struct whose children all + get dropped is dropped as well. + """ + fields = [pruned for field in schema if (pruned := _prune_field_without_id(field)) is not None] + return pa.schema(fields, metadata=schema.metadata) + + +def _prune_field_without_id(field: pa.Field) -> pa.Field | None: + if _get_field_id(field) is None: + return None + pruned_type = _prune_type_without_ids(field.type) + if pruned_type is None: + return None + return field.with_type(pruned_type) + + +def _prune_type_without_ids(arrow_type: pa.DataType) -> pa.DataType | None: + if pa.types.is_struct(arrow_type): + children = [arrow_type.field(i) for i in range(arrow_type.num_fields)] + pruned_children = [pruned for child in children if (pruned := _prune_field_without_id(child)) is not None] + return pa.struct(pruned_children) if pruned_children else None + elif isinstance(arrow_type, (pa.ListType, pa.LargeListType, pa.FixedSizeListType)): + element_field = _prune_field_without_id(arrow_type.value_field) + if element_field is None: + return None + if isinstance(arrow_type, pa.LargeListType): + return pa.large_list(element_field) + if isinstance(arrow_type, pa.FixedSizeListType): + return pa.list_(element_field, arrow_type.list_size) + return pa.list_(element_field) + elif isinstance(arrow_type, pa.MapType): + key_field = _prune_field_without_id(arrow_type.key_field) + value_field = _prune_field_without_id(arrow_type.item_field) + if key_field is None or value_field is None: + # Keeping a map without its value (or key) would change its type, so drop the whole map + return None + return pa.map_(key_field, value_field, keys_sorted=arrow_type.keys_sorted) + return arrow_type + + def _pyarrow_to_schema_without_ids( schema: pa.Schema, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, ) -> Schema: - result = visit_pyarrow( + return visit_pyarrow( schema, _ConvertToIcebergWithoutIDs(downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version), ) - assert isinstance(result, Schema) - return result def _pyarrow_schema_ensure_large_types(schema: pa.Schema) -> pa.Schema: @@ -1452,39 +1511,32 @@ def _get_field_id(field: pa.Field) -> int | None: class _HasIds(PyArrowSchemaVisitor[bool]): - """Return whether any field in the Arrow schema has an Iceberg field ID. - - Matches Java's ParquetSchemaUtil.hasIds, which is true if any column has an ID. - Fields without IDs are later pruned by _ConvertToIceberg. - """ - def schema(self, schema: pa.Schema, struct_result: bool) -> bool: return struct_result def struct(self, struct: pa.StructType, field_results: builtins.list[bool]) -> bool: - return any(field_results) + return all(field_results) def field(self, field: pa.Field, field_result: bool) -> bool: - return _get_field_id(field) is not None or field_result + return all([_get_field_id(field) is not None, field_result]) def list(self, list_type: pa.ListType, element_result: bool) -> bool: element_field = list_type.value_field element_id = _get_field_id(element_field) - return element_result or element_id is not None + return element_result and element_id is not None def map(self, map_type: pa.MapType, key_result: bool, value_result: bool) -> bool: key_field = map_type.key_field key_id = _get_field_id(key_field) value_field = map_type.item_field value_id = _get_field_id(value_field) - return any([key_id is not None, value_id is not None, key_result, value_result]) + return all([key_id is not None, value_id is not None, key_result, value_result]) def primitive(self, primitive: pa.DataType) -> bool: - # IDs live on pa.Field metadata, not on primitive types - return False + return True -class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema | None]): +class _ConvertToIceberg(PyArrowSchemaVisitor[IcebergType | Schema]): """Converts PyArrowSchema to Iceberg Schema.""" _field_names: builtins.list[str] @@ -1496,37 +1548,34 @@ def __init__( self._downcast_ns_timestamp_to_us = downcast_ns_timestamp_to_us self._format_version = format_version - def _field_id(self, field: pa.Field) -> int | None: - return _get_field_id(field) + def _field_id(self, field: pa.Field) -> int: + if (field_id := _get_field_id(field)) is not None: + return field_id + else: + raise ValueError(f"Cannot convert {field} to Iceberg Field as field_id is empty.") - def schema(self, schema: pa.Schema, struct_result: StructType | None) -> Schema: - return Schema(*(struct_result.fields if struct_result is not None else ())) + def schema(self, schema: pa.Schema, struct_result: StructType) -> Schema: + return Schema(*struct_result.fields) - def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField | None]) -> StructType | None: - fields = tuple(field for field in field_results if field is not None) - return StructType(*fields) if fields else None + def struct(self, struct: pa.StructType, field_results: builtins.list[NestedField]) -> StructType: + return StructType(*field_results) - def field(self, field: pa.Field, field_result: IcebergType | None) -> NestedField | None: + def field(self, field: pa.Field, field_result: IcebergType) -> NestedField: field_id = self._field_id(field) - if field_id is None or field_result is None: - return None field_doc = doc_str.decode() if (field.metadata and (doc_str := field.metadata.get(PYARROW_FIELD_DOC_KEY))) else None - return NestedField(field_id, field.name, field_result, required=not field.nullable, doc=field_doc) + field_type = field_result + return NestedField(field_id, field.name, field_type, required=not field.nullable, doc=field_doc) - def list(self, list_type: pa.ListType, element_result: IcebergType | None) -> ListType | None: + def list(self, list_type: pa.ListType, element_result: IcebergType) -> ListType: element_field = list_type.value_field element_id = self._field_id(element_field) - if element_id is None or element_result is None: - return None return ListType(element_id, element_result, element_required=not element_field.nullable) - def map(self, map_type: pa.MapType, key_result: IcebergType | None, value_result: IcebergType | None) -> MapType | None: + def map(self, map_type: pa.MapType, key_result: IcebergType, value_result: IcebergType) -> MapType: key_field = map_type.key_field key_id = self._field_id(key_field) value_field = map_type.item_field value_id = self._field_id(value_field) - if key_id is None or value_id is None or key_result is None or value_result is None: - return None return MapType(key_id, key_result, value_id, value_result, value_required=not value_field.nullable) def primitive(self, primitive: pa.DataType) -> PrimitiveType: @@ -1793,7 +1842,11 @@ def _task_to_record_batches( downcast_ns_timestamp_to_us if downcast_ns_timestamp_to_us is not None else format_version <= 2 ) file_schema = pyarrow_to_schema( - physical_schema, name_mapping, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=format_version + physical_schema, + name_mapping, + downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, + format_version=format_version, + prune_fields_without_ids=True, ) # Apply column projection rules: https://iceberg.apache.org/spec/#column-projection diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 489b3e765e..b73b0889f8 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3255,11 +3255,10 @@ def test_schema_compatible_missing_nullable_field_nested(table_schema_nested: Sc "person", pa.struct( [ - pa.field("age", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "17"}), + pa.field("age", pa.int32(), nullable=False), ] ), nullable=True, - metadata={"PARQUET:field_id": "15"}, ), ) try: @@ -3277,11 +3276,10 @@ def test_schema_mismatch_missing_required_field_nested(table_schema_nested: Sche "person", pa.struct( [ - pa.field("name", pa.string(), nullable=True, metadata={"PARQUET:field_id": "16"}), + pa.field("name", pa.string(), nullable=True), ] ), nullable=True, - metadata={"PARQUET:field_id": "15"}, ), ) expected = """Mismatch in fields: @@ -3344,6 +3342,41 @@ def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: _check_pyarrow_schema_compatible(table_schema_simple, other_schema) +def test_schema_mismatch_additional_field_without_id(table_schema_simple: Schema) -> None: + # Appending a table read from a data file (with field IDs) plus a new column without one + # must still be rejected, rather than dropping the new column + other_schema = pa.schema( + ( + pa.field("foo", pa.string(), nullable=True, metadata={"PARQUET:field_id": "1"}), + pa.field("bar", pa.int32(), nullable=False, metadata={"PARQUET:field_id": "2"}), + pa.field("baz", pa.bool_(), nullable=True, metadata={"PARQUET:field_id": "3"}), + pa.field("country", pa.string(), nullable=True), + ) + ) + + with pytest.raises( + ValueError, match=r"PyArrow table contains more columns: country. Update the schema first \(hint, use union_by_name\)." + ): + _check_pyarrow_schema_compatible(table_schema_simple, other_schema) + + +def test_projection_partial_field_ids(tmpdir: str) -> None: + # Columns without a field ID are ignored on read, even if their type is not supported + schema = Schema(NestedField(1, "foo", StringType(), required=False)) + arrow_schema = pa.schema( + [ + pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}), + pa.field("_row-id-col-922d6355", pa.time32("ms")), + ] + ) + table = pa.Table.from_arrays([pa.array(["a", "b", "c"]), pa.array([1, 2, 3], type=pa.time32("ms"))], schema=arrow_schema) + file = _write_table_to_file(f"{tmpdir}/partial-ids.parquet", arrow_schema, table) + + result = project(schema, [file]) + assert result.column_names == ["foo"] + assert result["foo"].to_pylist() == ["a", "b", "c"] + + def test_schema_compatible(table_schema_simple: Schema) -> None: try: _check_pyarrow_schema_compatible(table_schema_simple, table_schema_simple.as_arrow()) diff --git a/tests/io/test_pyarrow_visitor.py b/tests/io/test_pyarrow_visitor.py index 9d42634f8d..f176ae6a99 100644 --- a/tests/io/test_pyarrow_visitor.py +++ b/tests/io/test_pyarrow_visitor.py @@ -377,7 +377,7 @@ def test_simple_schema_has_missing_ids() -> None: assert not has_ids -def test_simple_schema_has_partial_ids() -> None: +def test_simple_schema_has_missing_ids_partial() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -386,7 +386,7 @@ def test_simple_schema_has_partial_ids() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert has_ids + assert not has_ids def test_nested_schema_has_missing_ids() -> None: @@ -436,7 +436,7 @@ def test_nested_schema_has_ids() -> None: assert has_ids -def test_nested_schema_has_partial_ids() -> None: +def test_nested_schema_has_partial_missing_ids() -> None: schema = pa.schema( [ pa.field("foo", pa.string(), nullable=False, metadata={"PARQUET:field_id": "1", "doc": "foo doc"}), @@ -457,24 +457,51 @@ def test_nested_schema_has_partial_ids() -> None: ) visitor = _HasIds() has_ids = visit_pyarrow(schema, visitor) - assert has_ids + assert not has_ids + + +def _id(field_id: int) -> dict[str, str]: + return {"PARQUET:field_id": str(field_id)} def test_pyarrow_to_schema_prunes_fields_without_ids() -> None: # A Databricks UniForm file may attach IDs only to data columns, leaving - # row-tracking columns unannotated. Previously _HasIds required every - # column to have an ID and fell back to name mapping, which failed when - # physical names differed from the mapped names. + # row-tracking columns unannotated. Without pruning, the file falls back to + # name mapping, which fails when physical names differ from the mapped names. file_schema = pa.schema( [ - pa.field("col-2b93b371", pa.string(), metadata={"PARQUET:field_id": "1"}), + pa.field("col-2b93b371", pa.string(), metadata=_id(1)), pa.field("_row-id-col-922d6355", pa.int64()), ] ) expected = Schema(NestedField(1, "col-2b93b371", StringType(), required=False)) name_mappings = [None, NameMapping([MappedField(field_id=1, names=["store_id"])])] for name_mapping in name_mappings: - assert pyarrow_to_schema(file_schema, name_mapping) == expected + assert pyarrow_to_schema(file_schema, name_mapping, prune_fields_without_ids=True) == expected + + +def test_pyarrow_to_schema_partial_ids_without_pruning() -> None: + # Without pruning (e.g. on write), partial IDs still fall back to the name mapping + file_schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("bar", pa.int64()), + ] + ) + with pytest.raises(ValueError, match="Parquet file does not have field-ids"): + pyarrow_to_schema(file_schema) + + +def test_pyarrow_to_schema_prunes_before_type_conversion() -> None: + # time32[ms] is not supported, but the column has no ID so it is never read + file_schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("bar", pa.time32("ms")), + ] + ) + expected = Schema(NestedField(1, "foo", StringType(), required=False)) + assert pyarrow_to_schema(file_schema, prune_fields_without_ids=True) == expected def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: @@ -484,7 +511,7 @@ def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: pa.field( "quux", pa.map_( - pa.field("key", pa.string(), nullable=False, metadata={"PARQUET:field_id": "7"}), + pa.field("key", pa.string(), nullable=False, metadata=_id(7)), pa.field( "value", pa.map_(pa.field("key", pa.string(), nullable=False), pa.field("value", pa.int32())), @@ -498,15 +525,15 @@ def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: "person", pa.struct( [ - pa.field("name", pa.string(), metadata={"PARQUET:field_id": "16"}), + pa.field("name", pa.string(), metadata=_id(16)), pa.field("age", pa.int32()), ] ), - metadata={"PARQUET:field_id": "15"}, + metadata=_id(15), ), ] ) - actual = str(pyarrow_to_schema(schema)) + actual = str(pyarrow_to_schema(schema, prune_fields_without_ids=True)) expected = """table { 1: foo: required string (foo doc) 15: person: optional struct<16: name: optional string> @@ -514,6 +541,71 @@ def test_pyarrow_to_schema_prunes_nested_fields_without_ids() -> None: assert actual == expected +def test_pyarrow_to_schema_prunes_list_without_element_id() -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field("no_element_id", pa.list_(pa.field("element", pa.int32())), metadata=_id(2)), + pa.field("with_element_id", pa.list_(pa.field("element", pa.int32(), metadata=_id(4))), metadata=_id(3)), + ] + ) + expected = Schema( + NestedField(1, "foo", StringType(), required=False), + NestedField(3, "with_element_id", ListType(4, IntegerType(), element_required=False), required=False), + ) + assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected + + +@pytest.mark.parametrize( + "key_metadata, value_metadata", + [ + (None, _id(4)), # value only + (_id(3), None), # key only, the value can't be kept on its own so the whole map goes + ], +) +def test_pyarrow_to_schema_prunes_map_without_key_or_value_id( + key_metadata: dict[str, str] | None, value_metadata: dict[str, str] | None +) -> None: + schema = pa.schema( + [ + pa.field("foo", pa.string(), metadata=_id(1)), + pa.field( + "partial_map", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata=key_metadata), + pa.field("value", pa.int32(), metadata=value_metadata), + ), + metadata=_id(2), + ), + pa.field( + "full_map", + pa.map_( + pa.field("key", pa.string(), nullable=False, metadata=_id(6)), + pa.field("value", pa.int32(), metadata=_id(7)), + ), + metadata=_id(5), + ), + ] + ) + expected = Schema( + NestedField(1, "foo", StringType(), required=False), + NestedField( + 5, + "full_map", + MapType(key_id=6, key_type=StringType(), value_id=7, value_type=IntegerType(), value_required=False), + required=False, + ), + ) + assert pyarrow_to_schema(schema, prune_fields_without_ids=True) == expected + + +def test_pyarrow_to_schema_without_any_ids_uses_name_mapping_when_pruning() -> None: + schema = pa.schema([pa.field("foo", pa.string())]) + name_mapping = NameMapping([MappedField(field_id=1, names=["foo"])]) + expected = Schema(NestedField(1, "foo", StringType(), required=False)) + assert pyarrow_to_schema(schema, name_mapping, prune_fields_without_ids=True) == expected + + def test_pyarrow_schema_to_schema_missing_ids_and_name_mapping(pyarrow_schema_simple_without_ids: pa.Schema) -> None: schema = pyarrow_schema_simple_without_ids with pytest.raises(ValueError) as exc_info: