Skip to content

Commit 03e80a7

Browse files
haizhou-zhaoclaude
andauthored
Skip unknown nested fields when resolving Avro read schemas (#4289)
Fields are resolved by ID, so a file field that is not in the read schema should be read and dropped. This already worked for primitives, but a file-only map, list, or struct raised ResolveError, e.g. when a writer adds extra map fields to data_file in manifests. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d570334 commit 03e80a7

2 files changed

Lines changed: 74 additions & 0 deletions

File tree

‎pyiceberg/avro/resolver.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,10 @@ def schema_partner(self, partner: IcebergType | None) -> IcebergType | None:
548548
raise ResolveError(f"File/read schema are not aligned for schema, got {partner}")
549549

550550
def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None:
551+
# A file-only field has no partner; its nested fields are read and dropped
552+
if partner is None:
553+
return None
554+
551555
if isinstance(partner, StructType):
552556
field = partner.field(field_id)
553557
else:
@@ -556,18 +560,27 @@ def field_partner(self, partner: IcebergType | None, field_id: int, field_name:
556560
return field.field_type if field else None
557561

558562
def list_element_partner(self, partner_list: IcebergType | None) -> IcebergType | None:
563+
if partner_list is None:
564+
return None
565+
559566
if isinstance(partner_list, ListType):
560567
return partner_list.element_type
561568

562569
raise ResolveError(f"File/read schema are not aligned for list, got {partner_list}")
563570

564571
def map_key_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
572+
if partner_map is None:
573+
return None
574+
565575
if isinstance(partner_map, MapType):
566576
return partner_map.key_type
567577

568578
raise ResolveError(f"File/read schema are not aligned for map, got {partner_map}")
569579

570580
def map_value_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
581+
if partner_map is None:
582+
return None
583+
571584
if isinstance(partner_map, MapType):
572585
return partner_map.value_type
573586

‎tests/avro/test_file.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,20 @@
4141
from pyiceberg.schema import Schema
4242
from pyiceberg.typedef import Record, TableVersion
4343
from pyiceberg.types import (
44+
BinaryType,
4445
BooleanType,
4546
DateType,
4647
DecimalType,
4748
DoubleType,
4849
FixedType,
4950
FloatType,
5051
IntegerType,
52+
ListType,
5153
LongType,
54+
MapType,
5255
NestedField,
5356
StringType,
57+
StructType,
5458
TimestampNanoType,
5559
TimestampType,
5660
TimestamptzNanoType,
@@ -347,6 +351,63 @@ def test_write_manifest_entry_with_fastavro_read_with_iceberg(format_version: Ta
347351
raise ValueError(f"Unsupported version: {format_version}")
348352

349353

354+
def test_read_manifest_entry_with_unknown_nested_fields() -> None:
355+
# Writers may add fields to data_file that the read schema does not know; they must be skipped by field ID
356+
read_schema = MANIFEST_ENTRY_SCHEMAS[2]
357+
data_file_fields = list(read_schema.find_field("data_file").field_type.fields)
358+
unknown_fields = [
359+
NestedField(1250, "unknown_map", MapType(1260, IntegerType(), 1270, BinaryType()), required=False),
360+
NestedField(1280, "unknown_list", ListType(1290, LongType()), required=False),
361+
NestedField(
362+
1300,
363+
"unknown_struct",
364+
StructType(NestedField(1310, "inner_map", MapType(1320, IntegerType(), 1330, StringType()), required=False)),
365+
required=False,
366+
),
367+
]
368+
# Place the unknown fields before known ones to check that the decoder skips their bytes
369+
pos = next(i for i, field in enumerate(data_file_fields) if field.name == "lower_bounds")
370+
file_data_file = StructType(*data_file_fields[:pos], *unknown_fields, *data_file_fields[pos:])
371+
file_schema = Schema(
372+
*[
373+
NestedField(field.field_id, field.name, file_data_file, required=field.required)
374+
if field.name == "data_file"
375+
else field
376+
for field in read_schema.fields
377+
]
378+
)
379+
380+
data_file = DataFile.from_args(
381+
content=DataFileContent.DATA,
382+
file_path="s3://some-path/some-file.parquet",
383+
file_format=FileFormat.PARQUET,
384+
partition=Record(),
385+
record_count=131327,
386+
file_size_in_bytes=220669226,
387+
lower_bounds={1: b"aaaaaaaaaaaaaaaa"},
388+
upper_bounds={1: b"zzzzzzzzzzzzzzzz"},
389+
sort_order_id=4,
390+
)
391+
entry = ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=8638475580105682862, data_file=data_file)
392+
393+
record = todict(entry)
394+
record["data_file"]["unknown_map"] = todict({1: b"\x01\x02"})
395+
record["data_file"]["unknown_list"] = [1, 2, 3]
396+
record["data_file"]["unknown_struct"] = {"inner_map": todict({2: "value"})}
397+
398+
with TemporaryDirectory() as tmpdir:
399+
tmp_avro_file = tmpdir + "/manifest_entry.avro"
400+
with open(tmp_avro_file, "wb") as out:
401+
writer(out, AvroSchemaConversion().iceberg_to_avro(file_schema, schema_name="manifest_entry"), [record])
402+
403+
with avro.AvroFile[ManifestEntry](
404+
input_file=PyArrowFileIO().new_input(tmp_avro_file),
405+
read_schema=read_schema,
406+
read_types={-1: ManifestEntry, 2: DataFile},
407+
) as avro_reader:
408+
assert list(avro_reader) == [entry]
409+
410+
350411
@pytest.mark.parametrize("is_required", [True, False])
351412
def test_all_primitive_types(is_required: bool) -> None:
352413
all_primitives_schema = Schema(

0 commit comments

Comments
 (0)