Skip to content

Commit 981be20

Browse files
haizhou-zhaoclaude
andcommitted
Skip unknown nested fields when resolving Avro read schemas
Fields are resolved by ID, so a file field that is not in the read schema should be read and dropped. This already worked for primitives, but a file-only map, list, or struct raised ResolveError, e.g. when a writer adds extra map fields to data_file in manifests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent e3564a5 commit 981be20

2 files changed

Lines changed: 74 additions & 0 deletions

File tree

‎pyiceberg/avro/resolver.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,10 @@ def schema_partner(self, partner: IcebergType | None) -> IcebergType | None:
548548
raise ResolveError(f"File/read schema are not aligned for schema, got {partner}")
549549

550550
def field_partner(self, partner: IcebergType | None, field_id: int, field_name: str) -> IcebergType | None:
551+
# A file-only field has no partner; its nested fields are read and dropped
552+
if partner is None:
553+
return None
554+
551555
if isinstance(partner, StructType):
552556
field = partner.field(field_id)
553557
else:
@@ -556,18 +560,27 @@ def field_partner(self, partner: IcebergType | None, field_id: int, field_name:
556560
return field.field_type if field else None
557561

558562
def list_element_partner(self, partner_list: IcebergType | None) -> IcebergType | None:
563+
if partner_list is None:
564+
return None
565+
559566
if isinstance(partner_list, ListType):
560567
return partner_list.element_type
561568

562569
raise ResolveError(f"File/read schema are not aligned for list, got {partner_list}")
563570

564571
def map_key_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
572+
if partner_map is None:
573+
return None
574+
565575
if isinstance(partner_map, MapType):
566576
return partner_map.key_type
567577

568578
raise ResolveError(f"File/read schema are not aligned for map, got {partner_map}")
569579

570580
def map_value_partner(self, partner_map: IcebergType | None) -> IcebergType | None:
581+
if partner_map is None:
582+
return None
583+
571584
if isinstance(partner_map, MapType):
572585
return partner_map.value_type
573586

‎tests/avro/test_file.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,20 @@
4141
from pyiceberg.schema import Schema
4242
from pyiceberg.typedef import Record, TableVersion
4343
from pyiceberg.types import (
44+
BinaryType,
4445
BooleanType,
4546
DateType,
4647
DecimalType,
4748
DoubleType,
4849
FixedType,
4950
FloatType,
5051
IntegerType,
52+
ListType,
5153
LongType,
54+
MapType,
5255
NestedField,
5356
StringType,
57+
StructType,
5458
TimestampNanoType,
5559
TimestampType,
5660
TimestamptzNanoType,
@@ -341,6 +345,63 @@ def test_write_manifest_entry_with_fastavro_read_with_iceberg(format_version: Ta
341345
raise ValueError(f"Unsupported version: {format_version}")
342346

343347

348+
def test_read_manifest_entry_with_unknown_nested_fields() -> None:
349+
# Writers may add fields to data_file that the read schema does not know; they must be skipped by field ID
350+
read_schema = MANIFEST_ENTRY_SCHEMAS[2]
351+
data_file_fields = list(read_schema.find_field("data_file").field_type.fields)
352+
unknown_fields = [
353+
NestedField(1250, "unknown_map", MapType(1260, IntegerType(), 1270, BinaryType()), required=False),
354+
NestedField(1280, "unknown_list", ListType(1290, LongType()), required=False),
355+
NestedField(
356+
1300,
357+
"unknown_struct",
358+
StructType(NestedField(1310, "inner_map", MapType(1320, IntegerType(), 1330, StringType()), required=False)),
359+
required=False,
360+
),
361+
]
362+
# Place the unknown fields before known ones to check that the decoder skips their bytes
363+
pos = next(i for i, field in enumerate(data_file_fields) if field.name == "lower_bounds")
364+
file_data_file = StructType(*data_file_fields[:pos], *unknown_fields, *data_file_fields[pos:])
365+
file_schema = Schema(
366+
*[
367+
NestedField(field.field_id, field.name, file_data_file, required=field.required)
368+
if field.name == "data_file"
369+
else field
370+
for field in read_schema.fields
371+
]
372+
)
373+
374+
data_file = DataFile.from_args(
375+
content=DataFileContent.DATA,
376+
file_path="s3://some-path/some-file.parquet",
377+
file_format=FileFormat.PARQUET,
378+
partition=Record(),
379+
record_count=131327,
380+
file_size_in_bytes=220669226,
381+
lower_bounds={1: b"aaaaaaaaaaaaaaaa"},
382+
upper_bounds={1: b"zzzzzzzzzzzzzzzz"},
383+
sort_order_id=4,
384+
)
385+
entry = ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=8638475580105682862, data_file=data_file)
386+
387+
record = todict(entry)
388+
record["data_file"]["unknown_map"] = todict({1: b"\x01\x02"})
389+
record["data_file"]["unknown_list"] = [1, 2, 3]
390+
record["data_file"]["unknown_struct"] = {"inner_map": todict({2: "value"})}
391+
392+
with TemporaryDirectory() as tmpdir:
393+
tmp_avro_file = tmpdir + "/manifest_entry.avro"
394+
with open(tmp_avro_file, "wb") as out:
395+
writer(out, AvroSchemaConversion().iceberg_to_avro(file_schema, schema_name="manifest_entry"), [record])
396+
397+
with avro.AvroFile[ManifestEntry](
398+
input_file=PyArrowFileIO().new_input(tmp_avro_file),
399+
read_schema=read_schema,
400+
read_types={-1: ManifestEntry, 2: DataFile},
401+
) as avro_reader:
402+
assert list(avro_reader) == [entry]
403+
404+
344405
@pytest.mark.parametrize("is_required", [True, False])
345406
def test_all_primitive_types(is_required: bool) -> None:
346407
all_primitives_schema = Schema(

0 commit comments

Comments
 (0)