Skip to content

Commit ab0afe7

Browse files
Avro: Cache the Avro to Iceberg schema conversion per schema string (#4384)
* Avro: Cache the Avro to Iceberg schema conversion per schema string * Avro: Cap the schema conversion cacher at 32 enteries
1 parent a385453 commit ab0afe7

2 files changed

Lines changed: 29 additions & 3 deletions

File tree

‎pyiceberg/avro/file.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from collections.abc import Callable
2626
from dataclasses import dataclass
2727
from enum import Enum
28+
from functools import lru_cache
2829
from types import TracebackType
2930
from typing import (
3031
Generic,
@@ -71,6 +72,11 @@
7172
_SCHEMA_KEY = "avro.schema"
7273

7374

75+
@lru_cache(maxsize=32)
76+
def _avro_schema_string_to_iceberg(avro_schema_string: str) -> Schema:
77+
return AvroSchemaConversion().avro_to_iceberg(json.loads(avro_schema_string))
78+
79+
7480
class AvroFileHeader(Record):
7581
@property
7682
def magic(self) -> bytes:
@@ -99,9 +105,7 @@ def compression_codec(self) -> type[Codec] | None:
99105

100106
def get_schema(self) -> Schema:
101107
if _SCHEMA_KEY in self.meta:
102-
avro_schema_string = self.meta[_SCHEMA_KEY]
103-
avro_schema = json.loads(avro_schema_string)
104-
return AvroSchemaConversion().avro_to_iceberg(avro_schema)
108+
return _avro_schema_string_to_iceberg(self.meta[_SCHEMA_KEY])
105109
else:
106110
raise ValueError("No schema found in Avro file headers")
107111

‎tests/avro/test_file.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,28 @@ def test_missing_schema() -> None:
9999
assert "No schema found in Avro file headers" in str(exc_info.value)
100100

101101

102+
def test_get_schema_reuses_conversion_for_identical_schema_strings() -> None:
103+
schema_string = (
104+
'{"type": "record", "name": "manifest_entry", "fields": ['
105+
'{"name": "status", "type": "int", "field-id": 0}, '
106+
'{"name": "snapshot_id", "type": ["null", "long"], "default": null, "field-id": 1}]}'
107+
)
108+
other_schema_string = (
109+
'{"type": "record", "name": "manifest_entry", "fields": [{"name": "status", "type": "int", "field-id": 0}]}'
110+
)
111+
112+
header = AvroFileHeader(bytes(0), {"avro.schema": schema_string}, bytes(16))
113+
same_schema_header = AvroFileHeader(bytes(0), {"avro.schema": schema_string}, bytes(16))
114+
other_schema_header = AvroFileHeader(bytes(0), {"avro.schema": other_schema_string}, bytes(16))
115+
116+
assert header.get_schema() == Schema(
117+
NestedField(field_id=0, name="status", field_type=IntegerType(), required=True),
118+
NestedField(field_id=1, name="snapshot_id", field_type=LongType(), required=False),
119+
)
120+
assert header.get_schema() is same_schema_header.get_schema()
121+
assert other_schema_header.get_schema() is not header.get_schema()
122+
123+
102124
# helper function to serialize our objects to dicts to enable
103125
# direct comparison with the dicts returned by fastavro
104126
def todict(obj: Any) -> Any:

0 commit comments

Comments
 (0)