From 83dcf852d3e40ea499329dcfddd6ef828110f8c5 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Tue, 1 Sep 2026 21:53:18 +0200 Subject: [PATCH 1/4] Add Parquet content-defined chunking (CDC) writer support PyArrow's ParquetWriter has supported content-defined chunking natively since 21.0.0, producing stable page boundaries across appends for content-addressable storage. Wire this through as write.parquet.content-defined-chunking.* table properties, mirroring the property names and defaults already used by iceberg-rust. PyArrow validates the chunk sizes itself and raises a clear error, so pyiceberg doesn't duplicate those checks. Requesting CDC on an older PyArrow raises an ImportError from a shared _require_pyarrow_version helper, which also replaces the existing Azure filesystem guard. --- mkdocs/docs/configuration.md | 4 ++ pyiceberg/io/pyarrow.py | 46 +++++++++++++++---- pyiceberg/table/__init__.py | 12 +++++ tests/io/test_pyarrow.py | 87 ++++++++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+), 8 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index eba97908b3..8274e93df9 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -85,6 +85,10 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | +| `write.parquet.content-defined-chunking.enabled` | Boolean | False | Enables content-defined chunking (CDC) for the Parquet writer, which produces stable page boundaries across appends. Requires `pyarrow>=21.0.0`, and raises at write time on older versions. | +| `write.parquet.content-defined-chunking.min-chunk-size` | Size in bytes | 256KiB (262144) | The minimum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.max-chunk-size` | Size in bytes | 1MiB (1048576) | The maximum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.norm-level` | Integer | 0 | The normalization level for content-defined chunking, controlling how tightly chunk sizes cluster around the average | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | | `write.avro.compression-codec` | `{gzip,zstd,snappy}` | gzip | Compression codec for manifest files and manifest lists. For `snappy`, install with `pip install pyiceberg[snappy]` | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 1dd816d5d3..23e9a04188 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -397,6 +397,13 @@ def to_input_file(self) -> PyArrowFile: return self +def _require_pyarrow_version(min_version: str, feature: str) -> None: + from packaging import version + + if version.parse(pyarrow.__version__) < version.parse(min_version): + raise ImportError(f"pyarrow version >= {min_version} required for {feature}, but found version {pyarrow.__version__}.") + + class PyArrowFileIO(FileIO): fs_by_scheme: Callable[[str, str | None], FileSystem] @@ -562,14 +569,7 @@ def _initialize_s3_fs(self, netloc: str | None) -> FileSystem: def _initialize_azure_fs(self, netloc: str | None = None) -> FileSystem: # https://arrow.apache.org/docs/python/generated/pyarrow.fs.AzureFileSystem.html - from packaging import version - - MIN_PYARROW_VERSION_SUPPORTING_AZURE_FS = "20.0.0" - if version.parse(pyarrow.__version__) < version.parse(MIN_PYARROW_VERSION_SUPPORTING_AZURE_FS): - raise ImportError( - f"pyarrow version >= {MIN_PYARROW_VERSION_SUPPORTING_AZURE_FS} required for AzureFileSystem support, " - f"but found version {pyarrow.__version__}." - ) + _require_pyarrow_version("20.0.0", "AzureFileSystem support") from pyarrow.fs import AzureFileSystem @@ -3058,6 +3058,36 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: ) elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties: warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2) + + # Unlike the properties above, which PyArrow's writer never supports and are safe to silently + # drop, CDC is a version-gated feature: silently ignoring it would produce a table that no longer + # has the content-defined chunk boundaries the user explicitly asked for, so this raises instead. + if property_as_bool( + properties=table_properties, + property_name=TableProperties.PARQUET_CDC_ENABLED, + default=TableProperties.PARQUET_CDC_ENABLED_DEFAULT, + ): + _require_pyarrow_version("21.0.0", "Parquet content-defined chunking") + # PyArrow itself validates these values (e.g. max-chunk-size > min-chunk-size) and raises a + # clear OSError, so there's no need to duplicate that validation here. + kwargs["use_content_defined_chunking"] = { + "min_chunk_size": property_as_int( + properties=table_properties, + property_name=TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE, + default=TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + ), + "max_chunk_size": property_as_int( + properties=table_properties, + property_name=TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE, + default=TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + ), + "norm_level": property_as_int( + properties=table_properties, + property_name=TableProperties.PARQUET_CDC_NORM_LEVEL, + default=TableProperties.PARQUET_CDC_NORM_LEVEL_DEFAULT, + ), + } + return kwargs diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index df7aaf824f..eb07413118 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -174,6 +174,18 @@ class TableProperties: PARQUET_BLOOM_FILTER_COLUMN_ENABLED_PREFIX = "write.parquet.bloom-filter-enabled.column" + PARQUET_CDC_ENABLED = "write.parquet.content-defined-chunking.enabled" + PARQUET_CDC_ENABLED_DEFAULT = False + + PARQUET_CDC_MIN_CHUNK_SIZE = "write.parquet.content-defined-chunking.min-chunk-size" + PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT = 256 * 1024 # 256 KiB + + PARQUET_CDC_MAX_CHUNK_SIZE = "write.parquet.content-defined-chunking.max-chunk-size" + PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT = 1024 * 1024 # 1 MiB + + PARQUET_CDC_NORM_LEVEL = "write.parquet.content-defined-chunking.norm-level" + PARQUET_CDC_NORM_LEVEL_DEFAULT = 0 + WRITE_TARGET_FILE_SIZE_BYTES = "write.target-file-size-bytes" WRITE_TARGET_FILE_SIZE_BYTES_DEFAULT = 512 * 1024 * 1024 # 512 MB diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index abb7d384d5..4b5073da6d 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -81,6 +81,7 @@ _check_pyarrow_schema_compatible, _ConvertToArrowSchema, _determine_partitions, + _get_parquet_writer_kwargs, _primitive_to_physical, _read_all_delete_files, _read_deletes, @@ -6102,3 +6103,89 @@ def test_dictionary_columns_produces_dict_encoded_output(tmpdir: str) -> None: # Values must be identical assert result_plain.column("label").to_pylist() == result_dict.column("label").to_pylist() + + +@pytest.mark.parametrize( + "table_properties,expected", + [ + ({}, None), + ( + {TableProperties.PARQUET_CDC_ENABLED: "true"}, + { + "min_chunk_size": TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + "max_chunk_size": TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + "norm_level": TableProperties.PARQUET_CDC_NORM_LEVEL_DEFAULT, + }, + ), + ( + { + TableProperties.PARQUET_CDC_ENABLED: "true", + TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE: "4096", + TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE: "8192", + TableProperties.PARQUET_CDC_NORM_LEVEL: "2", + }, + {"min_chunk_size": 4096, "max_chunk_size": 8192, "norm_level": 2}, + ), + ], +) +def test_get_parquet_writer_kwargs_cdc(table_properties: dict[str, str], expected: dict[str, int] | None) -> None: + kwargs = _get_parquet_writer_kwargs(table_properties) + assert kwargs.get("use_content_defined_chunking") == expected + + +def test_get_parquet_writer_kwargs_cdc_enabled_unsupported_pyarrow_version(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(pyarrow, "__version__", "17.0.0") + with pytest.raises(ImportError, match="pyarrow version >= 21.0.0"): + _get_parquet_writer_kwargs({TableProperties.PARQUET_CDC_ENABLED: "true"}) + + +def test_get_parquet_writer_kwargs_cdc_invalid_chunk_sizes_raises_from_pyarrow() -> None: + """PyArrow validates min/max chunk sizes itself; pyiceberg doesn't duplicate that check.""" + kwargs = _get_parquet_writer_kwargs( + { + TableProperties.PARQUET_CDC_ENABLED: "true", + TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE: "8192", + TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE: "4096", + } + ) + table = pa.table({"id": pa.array([1, 2, 3], type=pa.int32())}) + with pytest.raises(pa.ArrowIOError, match="max_chunk_size"): + with pq.ParquetWriter(pa.BufferOutputStream(), table.schema, **kwargs) as writer: + writer.write_table(table) + + +def test_write_file_with_content_defined_chunking_enabled(tmp_path: Path) -> None: + """Writing a table with CDC enabled should forward use_content_defined_chunking to pq.ParquetWriter.""" + from pyiceberg.table import WriteTask + + table_schema = Schema(NestedField(1, "id", IntegerType(), required=False)) + arrow_data = pa.table({"id": pa.array(range(1000), type=pa.int32())}) + + table_metadata = TableMetadataV2( + location=f"file://{tmp_path}", + last_column_id=1, + format_version=2, + schemas=[table_schema], + partition_specs=[PartitionSpec()], + properties={TableProperties.PARQUET_CDC_ENABLED: "true"}, + ) + + task = WriteTask( + write_uuid=uuid.uuid4(), + task_id=0, + record_batches=arrow_data.to_batches(), + schema=table_schema, + ) + + with patch("pyiceberg.io.pyarrow.pq.ParquetWriter", wraps=pq.ParquetWriter) as mock_writer: + data_files = list(write_file(io=PyArrowFileIO(), table_metadata=table_metadata, tasks=iter([task]))) + + assert mock_writer.call_args.kwargs["use_content_defined_chunking"] == { + "min_chunk_size": TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, + "max_chunk_size": TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, + "norm_level": TableProperties.PARQUET_CDC_NORM_LEVEL_DEFAULT, + } + + assert len(data_files) == 1 + written_table = pq.read_table(data_files[0].file_path.replace("file://", "")) + assert written_table.column("id").to_pylist() == list(range(1000)) From 8d042840abc60689019a6e0ffeaaa44188b801ca Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Mon, 14 Sep 2026 14:56:02 +0200 Subject: [PATCH 2/4] Skip CDC tests on pyarrow < 21, fix stale comment The CDC tests raised ImportError on pyarrow 18-20, which pyproject still declares as supported. --- mkdocs/docs/configuration.md | 6 +++--- pyiceberg/io/pyarrow.py | 8 +++----- tests/io/test_pyarrow.py | 13 +++++++++++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 8274e93df9..7fe5f11e10 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -85,9 +85,9 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.page-size-bytes` | Size in bytes | 1MB | Set a target threshold for the approximate encoded size of data pages within a column chunk | | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | -| `write.parquet.content-defined-chunking.enabled` | Boolean | False | Enables content-defined chunking (CDC) for the Parquet writer, which produces stable page boundaries across appends. Requires `pyarrow>=21.0.0`, and raises at write time on older versions. | -| `write.parquet.content-defined-chunking.min-chunk-size` | Size in bytes | 256KiB (262144) | The minimum chunk size used for content-defined chunking | -| `write.parquet.content-defined-chunking.max-chunk-size` | Size in bytes | 1MiB (1048576) | The maximum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.enabled` | Boolean | False | Enables content-defined chunking (CDC) for the Parquet writer, which produces stable page boundaries across appends. Requires `pyarrow>=21.0.0`; raises at write time on older versions. | +| `write.parquet.content-defined-chunking.min-chunk-size` | Size in bytes | 256KiB | The minimum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.max-chunk-size` | Size in bytes | 1MiB | The maximum chunk size used for content-defined chunking | | `write.parquet.content-defined-chunking.norm-level` | Integer | 0 | The normalization level for content-defined chunking, controlling how tightly chunk sizes cluster around the average | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported | diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 23e9a04188..a7e42fd630 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3059,17 +3059,15 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties: warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2) - # Unlike the properties above, which PyArrow's writer never supports and are safe to silently - # drop, CDC is a version-gated feature: silently ignoring it would produce a table that no longer - # has the content-defined chunk boundaries the user explicitly asked for, so this raises instead. + # Unlike the unsupported options warned about above, a CDC request must not be dropped: + # writing without the requested chunk boundaries silently defeats the point, so raise instead. if property_as_bool( properties=table_properties, property_name=TableProperties.PARQUET_CDC_ENABLED, default=TableProperties.PARQUET_CDC_ENABLED_DEFAULT, ): _require_pyarrow_version("21.0.0", "Parquet content-defined chunking") - # PyArrow itself validates these values (e.g. max-chunk-size > min-chunk-size) and raises a - # clear OSError, so there's no need to duplicate that validation here. + # PyArrow validates these values itself (e.g. max-chunk-size > min-chunk-size). kwargs["use_content_defined_chunking"] = { "min_chunk_size": property_as_int( properties=table_properties, diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 4b5073da6d..cf26558446 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -142,6 +142,11 @@ reason="Requires pyarrow version >= 20.0.0", ) +skip_if_pyarrow_too_old_for_cdc = pytest.mark.skipif( + version.parse(pyarrow.__version__) < version.parse("21.0.0"), + reason="Requires pyarrow version >= 21.0.0", +) + def test_pyarrow_infer_local_fs_from_path() -> None: """Test path with `file` scheme and no scheme both use LocalFileSystem""" @@ -6109,15 +6114,16 @@ def test_dictionary_columns_produces_dict_encoded_output(tmpdir: str) -> None: "table_properties,expected", [ ({}, None), - ( + pytest.param( {TableProperties.PARQUET_CDC_ENABLED: "true"}, { "min_chunk_size": TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE_DEFAULT, "max_chunk_size": TableProperties.PARQUET_CDC_MAX_CHUNK_SIZE_DEFAULT, "norm_level": TableProperties.PARQUET_CDC_NORM_LEVEL_DEFAULT, }, + marks=skip_if_pyarrow_too_old_for_cdc, ), - ( + pytest.param( { TableProperties.PARQUET_CDC_ENABLED: "true", TableProperties.PARQUET_CDC_MIN_CHUNK_SIZE: "4096", @@ -6125,6 +6131,7 @@ def test_dictionary_columns_produces_dict_encoded_output(tmpdir: str) -> None: TableProperties.PARQUET_CDC_NORM_LEVEL: "2", }, {"min_chunk_size": 4096, "max_chunk_size": 8192, "norm_level": 2}, + marks=skip_if_pyarrow_too_old_for_cdc, ), ], ) @@ -6139,6 +6146,7 @@ def test_get_parquet_writer_kwargs_cdc_enabled_unsupported_pyarrow_version(monke _get_parquet_writer_kwargs({TableProperties.PARQUET_CDC_ENABLED: "true"}) +@skip_if_pyarrow_too_old_for_cdc def test_get_parquet_writer_kwargs_cdc_invalid_chunk_sizes_raises_from_pyarrow() -> None: """PyArrow validates min/max chunk sizes itself; pyiceberg doesn't duplicate that check.""" kwargs = _get_parquet_writer_kwargs( @@ -6154,6 +6162,7 @@ def test_get_parquet_writer_kwargs_cdc_invalid_chunk_sizes_raises_from_pyarrow() writer.write_table(table) +@skip_if_pyarrow_too_old_for_cdc def test_write_file_with_content_defined_chunking_enabled(tmp_path: Path) -> None: """Writing a table with CDC enabled should forward use_content_defined_chunking to pq.ParquetWriter.""" from pyiceberg.table import WriteTask From a861417b20924b32862f177c46d3090cd2e1ed04 Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Tue, 15 Sep 2026 10:00:21 +0200 Subject: [PATCH 3/4] Remove unhelpful comment per review feedback --- pyiceberg/io/pyarrow.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index a7e42fd630..d0271d2c3e 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3059,8 +3059,6 @@ def _get_parquet_writer_kwargs(table_properties: Properties) -> dict[str, Any]: elif TableProperties.PARQUET_PAGE_ROW_LIMIT in table_properties: warnings.warn(f"{TableProperties.PARQUET_PAGE_ROW_LIMIT} requires pyarrow >= 23.0.0", stacklevel=2) - # Unlike the unsupported options warned about above, a CDC request must not be dropped: - # writing without the requested chunk boundaries silently defeats the point, so raise instead. if property_as_bool( properties=table_properties, property_name=TableProperties.PARQUET_CDC_ENABLED, From 6c47dd6bb4373a8fcb630e936bb217f25bde08cd Mon Sep 17 00:00:00 2001 From: Krisztian Szucs Date: Wed, 23 Sep 2026 18:52:07 +0200 Subject: [PATCH 4/4] chore: use byte values for CDC chunk size defaults in docs --- mkdocs/docs/configuration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 7fe5f11e10..f45686e73a 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -86,8 +86,8 @@ Iceberg tables support table properties to configure table behavior. | `write.parquet.page-row-limit` | Number of rows | 20000 | Set a target threshold for the maximum number of rows within a column chunk | | `write.parquet.dict-size-bytes` | Size in bytes | 2MB | Set the dictionary page size limit per row group | | `write.parquet.content-defined-chunking.enabled` | Boolean | False | Enables content-defined chunking (CDC) for the Parquet writer, which produces stable page boundaries across appends. Requires `pyarrow>=21.0.0`; raises at write time on older versions. | -| `write.parquet.content-defined-chunking.min-chunk-size` | Size in bytes | 256KiB | The minimum chunk size used for content-defined chunking | -| `write.parquet.content-defined-chunking.max-chunk-size` | Size in bytes | 1MiB | The maximum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.min-chunk-size` | Size in bytes | 262144 | The minimum chunk size used for content-defined chunking | +| `write.parquet.content-defined-chunking.max-chunk-size` | Size in bytes | 1048576 | The maximum chunk size used for content-defined chunking | | `write.parquet.content-defined-chunking.norm-level` | Integer | 0 | The normalization level for content-defined chunking, controlling how tightly chunk sizes cluster around the average | | `write.target-file-size-bytes` | Size in bytes | 536870912 (512MB) | Target size of generated data files. Measured in uncompressed in-memory Arrow bytes, so files on disk are typically smaller after compression | | `write.format.default` | `{parquet}` | parquet | File format for data files written by PyIceberg. Only Parquet is supported |