Skip to content

Commit 0870ff8

Browse files
committed
fix: map write.parquet.page-row-limit to pyarrow's max_rows_per_page
1 parent 7d51e52 commit 0870ff8

2 files changed

Lines changed: 10 additions & 4 deletions

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2986,7 +2986,7 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]:
29862986
if compression_codec == ICEBERG_UNCOMPRESSED_CODEC:
29872987
compression_codec = PYARROW_UNCOMPRESSED_CODEC
29882988

2989-
return {
2989+
kwargs: dict[str, Any] = {
29902990
"compression": compression_codec,
29912991
"compression_level": compression_level,
29922992
"data_page_size": property_as_int(
@@ -2999,12 +2999,17 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]:
29992999
property_name=TableProperties.PARQUET_DICT_SIZE_BYTES,
30003000
default=TableProperties.PARQUET_DICT_SIZE_BYTES_DEFAULT,
30013001
),
3002-
"write_batch_size": property_as_int(
3002+
}
3003+
# pyarrow added max_rows_per_page, the per-page row limit, in 23.0.0.
3004+
if tuple(map(int, pa.__version__.split(".")[:2])) >= (23, 0):
3005+
kwargs["max_rows_per_page"] = property_as_int(
30033006
properties=table_properties,
30043007
property_name=TableProperties.PARQUET_PAGE_ROW_LIMIT,
30053008
default=TableProperties.PARQUET_PAGE_ROW_LIMIT_DEFAULT,
3006-
),
3007-
}
3009+
)
3010+
elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties:
3011+
warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2)
3012+
return kwargs
30083013

30093014

30103015
def _dataframe_to_data_files(

‎tests/integration/test_writes/test_writes.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -669,6 +669,7 @@ def test_write_parquet_compression_properties(
669669
[
670670
({"write.parquet.page-size-bytes": "42"}, {"data_page_size": 42}),
671671
({"write.parquet.dict-size-bytes": "42"}, {"dictionary_pagesize_limit": 42}),
672+
({"write.parquet.page-row-limit": "42"}, {"max_rows_per_page": 42, "write_batch_size": None}),
672673
],
673674
)
674675
def test_write_parquet_other_properties(

0 commit comments

Comments
 (0)