|
41 | 41 | from pyiceberg.schema import Schema |
42 | 42 | from pyiceberg.typedef import Record, TableVersion |
43 | 43 | from pyiceberg.types import ( |
| 44 | + BinaryType, |
44 | 45 | BooleanType, |
45 | 46 | DateType, |
46 | 47 | DecimalType, |
47 | 48 | DoubleType, |
48 | 49 | FixedType, |
49 | 50 | FloatType, |
50 | 51 | IntegerType, |
| 52 | + ListType, |
51 | 53 | LongType, |
| 54 | + MapType, |
52 | 55 | NestedField, |
53 | 56 | StringType, |
| 57 | + StructType, |
54 | 58 | TimestampNanoType, |
55 | 59 | TimestampType, |
56 | 60 | TimestamptzNanoType, |
@@ -341,6 +345,63 @@ def test_write_manifest_entry_with_fastavro_read_with_iceberg(format_version: Ta |
341 | 345 | raise ValueError(f"Unsupported version: {format_version}") |
342 | 346 |
|
343 | 347 |
|
| 348 | +def test_read_manifest_entry_with_unknown_nested_fields() -> None: |
| 349 | + # Writers may add fields to data_file that the read schema does not know; they must be skipped by field ID |
| 350 | + read_schema = MANIFEST_ENTRY_SCHEMAS[2] |
| 351 | + data_file_fields = list(read_schema.find_field("data_file").field_type.fields) |
| 352 | + unknown_fields = [ |
| 353 | + NestedField(1250, "unknown_map", MapType(1260, IntegerType(), 1270, BinaryType()), required=False), |
| 354 | + NestedField(1280, "unknown_list", ListType(1290, LongType()), required=False), |
| 355 | + NestedField( |
| 356 | + 1300, |
| 357 | + "unknown_struct", |
| 358 | + StructType(NestedField(1310, "inner_map", MapType(1320, IntegerType(), 1330, StringType()), required=False)), |
| 359 | + required=False, |
| 360 | + ), |
| 361 | + ] |
| 362 | + # Place the unknown fields before known ones to check that the decoder skips their bytes |
| 363 | + pos = next(i for i, field in enumerate(data_file_fields) if field.name == "lower_bounds") |
| 364 | + file_data_file = StructType(*data_file_fields[:pos], *unknown_fields, *data_file_fields[pos:]) |
| 365 | + file_schema = Schema( |
| 366 | + *[ |
| 367 | + NestedField(field.field_id, field.name, file_data_file, required=field.required) |
| 368 | + if field.name == "data_file" |
| 369 | + else field |
| 370 | + for field in read_schema.fields |
| 371 | + ] |
| 372 | + ) |
| 373 | + |
| 374 | + data_file = DataFile.from_args( |
| 375 | + content=DataFileContent.DATA, |
| 376 | + file_path="s3://some-path/some-file.parquet", |
| 377 | + file_format=FileFormat.PARQUET, |
| 378 | + partition=Record(), |
| 379 | + record_count=131327, |
| 380 | + file_size_in_bytes=220669226, |
| 381 | + lower_bounds={1: b"aaaaaaaaaaaaaaaa"}, |
| 382 | + upper_bounds={1: b"zzzzzzzzzzzzzzzz"}, |
| 383 | + sort_order_id=4, |
| 384 | + ) |
| 385 | + entry = ManifestEntry.from_args(status=ManifestEntryStatus.ADDED, snapshot_id=8638475580105682862, data_file=data_file) |
| 386 | + |
| 387 | + record = todict(entry) |
| 388 | + record["data_file"]["unknown_map"] = todict({1: b"\x01\x02"}) |
| 389 | + record["data_file"]["unknown_list"] = [1, 2, 3] |
| 390 | + record["data_file"]["unknown_struct"] = {"inner_map": todict({2: "value"})} |
| 391 | + |
| 392 | + with TemporaryDirectory() as tmpdir: |
| 393 | + tmp_avro_file = tmpdir + "/manifest_entry.avro" |
| 394 | + with open(tmp_avro_file, "wb") as out: |
| 395 | + writer(out, AvroSchemaConversion().iceberg_to_avro(file_schema, schema_name="manifest_entry"), [record]) |
| 396 | + |
| 397 | + with avro.AvroFile[ManifestEntry]( |
| 398 | + input_file=PyArrowFileIO().new_input(tmp_avro_file), |
| 399 | + read_schema=read_schema, |
| 400 | + read_types={-1: ManifestEntry, 2: DataFile}, |
| 401 | + ) as avro_reader: |
| 402 | + assert list(avro_reader) == [entry] |
| 403 | + |
| 404 | + |
344 | 405 | @pytest.mark.parametrize("is_required", [True, False]) |
345 | 406 | def test_all_primitive_types(is_required: bool) -> None: |
346 | 407 | all_primitives_schema = Schema( |
|
0 commit comments