From ec1eb10613e19fda96399ce3f6a9d249b22264c9 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 13:15:49 +0530 Subject: [PATCH 1/4] Report NaN value counts for written Parquet files Data files written by the PyArrow writer always had an empty nan_value_counts map, because the counts were never computed from the Parquet metadata. The scan planner supports them, so files written by pyiceberg could never be pruned on is_nan, not_nan, or range filters, while files written by parquet-mr could. PyArrow does not expose a NaN count on its Parquet column statistics, so count the non-null NaNs of the top-level float and double columns in the Arrow table the writer receives, and merge them into the statistics of the file. Columns that cannot be counted are left out of the map, which only keeps them from being pruned. --- pyiceberg/io/pyarrow.py | 54 ++++++++++++++++++++++++++++++++++++---- tests/io/test_pyarrow.py | 33 ++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 53db9cd6b7..fd7048545f 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -39,7 +39,7 @@ import warnings from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator -from dataclasses import dataclass +from dataclasses import dataclass, replace from decimal import Decimal from enum import Enum from functools import lru_cache, singledispatch @@ -2717,6 +2717,45 @@ def parquet_path_to_id_mapping( return result +def _nan_value_counts_from_arrow(table: pa.Table, schema: Schema) -> dict[int, int]: + """Count the non-null NaN values of the top-level float and double columns. + + PyArrow does not expose the NaN count of a Parquet column statistic, so the counts are taken + from the data before it is written. Columns that cannot be counted are left out, which only + means the file cannot be pruned on NaN counts. + + Args: + table (pyarrow.Table): The data that is written to the file. + schema (Schema): The schema of the file. + + Returns: + A mapping of Iceberg field ID to the number of non-null NaN values. + """ + nan_value_counts: dict[int, int] = {} + for pos, arrow_field in enumerate(table.schema): + if (field_id := _get_field_id(arrow_field)) is None: + continue + + try: + field_type = schema.find_field(field_id).field_type + except ValueError: + continue + + if not isinstance(field_type, (FloatType, DoubleType)): + continue + + try: + count = pc.sum(pc.is_nan(table.column(pos)), min_count=0) + except (pyarrow.lib.ArrowInvalid, pyarrow.lib.ArrowNotImplementedError) as e: + logger.warning("Could not count NaN values for column %s: %s", arrow_field.name, e) + continue + + if count is not None: + nan_value_counts[field_id] = count.as_py() + + return nan_value_counts + + def data_file_statistics_from_parquet_metadata( parquet_metadata: pq.FileMetaData, stats_columns: dict[int, StatisticsCollector], @@ -2868,6 +2907,7 @@ def __init__( self._fos: OutputStream | None = None self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties) self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata) + self._nan_value_counts: dict[int, int] = {} self._row_group_size = property_as_int( properties=properties, property_name=TableProperties.PARQUET_ROW_GROUP_LIMIT, @@ -2889,6 +2929,7 @@ def write(self, table: pa.Table) -> None: raise self._fos = fos self._writer.write(table, row_group_size=self._row_group_size) + self._nan_value_counts.update(_nan_value_counts_from_arrow(table, self._file_schema)) def close(self) -> DataFileStatistics: if self._result is not None: @@ -2897,10 +2938,13 @@ def close(self) -> DataFileStatistics: raise ValueError("Cannot close a writer that was never written to") with self._fos: self._writer.close() - self._result = data_file_statistics_from_parquet_metadata( - parquet_metadata=self._writer.writer.metadata, - stats_columns=compute_statistics_plan(self._file_schema, self._properties), - parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), + self._result = replace( + data_file_statistics_from_parquet_metadata( + parquet_metadata=self._writer.writer.metadata, + stats_columns=compute_statistics_plan(self._file_schema, self._properties), + parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), + ), + nan_value_counts=self._nan_value_counts, ) return self._result diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index b0f1cdde37..9341cfdd46 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3596,6 +3596,39 @@ def test_append_to_table_bucketed_by_time(catalog: InMemoryCatalog) -> None: assert table.scan(row_filter=EqualTo("event_time", "22:31:08")).to_arrow()["id"].to_pylist() == [2] +def test_append_counts_nan_values(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(id=1, name="f", field_type=FloatType(), required=False), + NestedField(id=2, name="d", field_type=DoubleType(), required=False), + ) + catalog.create_namespace("default") + table = catalog.create_table("default.test_append_nan_counts", schema=schema) + + table.append( + pa.table( + { + "f": pa.array([float("nan"), float("nan"), 1.0, None], type=pa.float32()), + "d": pa.array([float("nan"), 2.0, None, float("nan")], type=pa.float64()), + } + ) + ) + + data_file = next(iter(table.scan().plan_files())).file + assert dict(data_file.nan_value_counts) == {1: 2, 2: 2} + + +def test_scan_prunes_files_without_nan_values(catalog: InMemoryCatalog) -> None: + schema = Schema(NestedField(id=1, name="d", field_type=DoubleType(), required=False)) + catalog.create_namespace("default") + table = catalog.create_table("default.test_scan_prunes_nan_counts", schema=schema) + + table.append(pa.table({"d": pa.array([float("nan"), 1.0], type=pa.float64())})) + table.append(pa.table({"d": pa.array([2.0, 3.0], type=pa.float64())})) + + assert len(list(table.scan(row_filter="d IS NAN").plan_files())) == 1 + assert len(list(table.scan(row_filter="d IS NOT NAN").plan_files())) == 2 + + def test_inspect_partition_for_nested_field(catalog: InMemoryCatalog) -> None: schema = Schema( NestedField(id=1, name="foo", field_type=StringType(), required=True), From 7c3caf546edd2f38d3533798f37a228c601d5216 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Tue, 17 Nov 2026 22:16:40 +0530 Subject: [PATCH 2/4] Expect NaN counts on written files in the merge manifest test --- tests/integration/test_writes/test_writes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_writes/test_writes.py b/tests/integration/test_writes/test_writes.py index fe720440fb..e0d647c2e9 100644 --- a/tests/integration/test_writes/test_writes.py +++ b/tests/integration/test_writes/test_writes.py @@ -1688,7 +1688,7 @@ def test_merge_manifests_file_content(session_catalog: Catalog, arrow_table_with (11, b"\x01"), (12, b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"), ] - assert tbl_a_data_file["nan_value_counts"] == [] + assert tbl_a_data_file["nan_value_counts"] == [(6, 0), (7, 0)] assert tbl_a_data_file["null_value_counts"] == [ (1, 1), (2, 1), From f1d81bad2d2da978aaf2dc0b0f1a0c22807694fc Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 00:01:59 +0530 Subject: [PATCH 3/4] fix(pyarrow): accumulate NaN counts and respect metrics mode --- pyiceberg/io/pyarrow.py | 40 ++++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index fd7048545f..42fc789679 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -39,7 +39,7 @@ import warnings from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Iterator -from dataclasses import dataclass, replace +from dataclasses import dataclass from decimal import Decimal from enum import Enum from functools import lru_cache, singledispatch @@ -2717,7 +2717,7 @@ def parquet_path_to_id_mapping( return result -def _nan_value_counts_from_arrow(table: pa.Table, schema: Schema) -> dict[int, int]: +def _nan_value_counts_from_arrow(table: pa.Table, stats_columns: dict[int, StatisticsCollector]) -> dict[int, int]: """Count the non-null NaN values of the top-level float and double columns. PyArrow does not expose the NaN count of a Parquet column statistic, so the counts are taken @@ -2726,7 +2726,7 @@ def _nan_value_counts_from_arrow(table: pa.Table, schema: Schema) -> dict[int, i Args: table (pyarrow.Table): The data that is written to the file. - schema (Schema): The schema of the file. + stats_columns (dict[int, StatisticsCollector]): The statistics gathering plan. Returns: A mapping of Iceberg field ID to the number of non-null NaN values. @@ -2736,22 +2736,17 @@ def _nan_value_counts_from_arrow(table: pa.Table, schema: Schema) -> dict[int, i if (field_id := _get_field_id(arrow_field)) is None: continue - try: - field_type = schema.find_field(field_id).field_type - except ValueError: + stats_col = stats_columns.get(field_id) + if stats_col is None: continue - if not isinstance(field_type, (FloatType, DoubleType)): + if stats_col.mode == MetricsMode(MetricModeTypes.NONE): continue - try: - count = pc.sum(pc.is_nan(table.column(pos)), min_count=0) - except (pyarrow.lib.ArrowInvalid, pyarrow.lib.ArrowNotImplementedError) as e: - logger.warning("Could not count NaN values for column %s: %s", arrow_field.name, e) + if not isinstance(stats_col.iceberg_type, (FloatType, DoubleType)): continue - if count is not None: - nan_value_counts[field_id] = count.as_py() + nan_value_counts[field_id] = pc.sum(pc.is_nan(table.column(pos)), min_count=0).as_py() return nan_value_counts @@ -2760,6 +2755,7 @@ def data_file_statistics_from_parquet_metadata( parquet_metadata: pq.FileMetaData, stats_columns: dict[int, StatisticsCollector], parquet_column_mapping: dict[str, int], + nan_value_counts: dict[int, int] | None = None, ) -> DataFileStatistics: """ Compute and return DataFileStatistics that includes the following. @@ -2777,13 +2773,14 @@ def data_file_statistics_from_parquet_metadata( stats_columns (Dict[int, StatisticsCollector]): The statistics gathering plan. It is required to set the mode for column metrics collection parquet_column_mapping (Dict[str, int]): The mapping of the parquet file name to the field ID + nan_value_counts (dict[int, int] | None): The NaN value counts computed from the data, if available """ column_sizes: dict[int, int] = {} value_counts: dict[int, int] = {} split_offsets: list[int] = [] null_value_counts: dict[int, int] = {} - nan_value_counts: dict[int, int] = {} + nan_value_counts = dict(nan_value_counts or {}) col_aggs = {} @@ -2902,11 +2899,11 @@ def __init__( ) -> None: self._output_file = output_file self._file_schema = file_schema - self._properties = properties self._writer: pq.ParquetWriter | None = None self._fos: OutputStream | None = None self._parquet_writer_kwargs = _get_parquet_writer_kwargs(properties) self._parquet_writer_kwargs["encryption_properties"] = _parquet_encryption_properties(key_metadata) + self._stats_columns = compute_statistics_plan(file_schema, properties) self._nan_value_counts: dict[int, int] = {} self._row_group_size = property_as_int( properties=properties, @@ -2929,7 +2926,8 @@ def write(self, table: pa.Table) -> None: raise self._fos = fos self._writer.write(table, row_group_size=self._row_group_size) - self._nan_value_counts.update(_nan_value_counts_from_arrow(table, self._file_schema)) + for field_id, count in _nan_value_counts_from_arrow(table, self._stats_columns).items(): + self._nan_value_counts[field_id] = self._nan_value_counts.get(field_id, 0) + count def close(self) -> DataFileStatistics: if self._result is not None: @@ -2938,12 +2936,10 @@ def close(self) -> DataFileStatistics: raise ValueError("Cannot close a writer that was never written to") with self._fos: self._writer.close() - self._result = replace( - data_file_statistics_from_parquet_metadata( - parquet_metadata=self._writer.writer.metadata, - stats_columns=compute_statistics_plan(self._file_schema, self._properties), - parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), - ), + self._result = data_file_statistics_from_parquet_metadata( + parquet_metadata=self._writer.writer.metadata, + stats_columns=self._stats_columns, + parquet_column_mapping=parquet_path_to_id_mapping(self._file_schema), nan_value_counts=self._nan_value_counts, ) return self._result From 4223933d0b6002fe3f3db4d19ab352fa46f79c66 Mon Sep 17 00:00:00 2001 From: Anurag Singh Date: Wed, 18 Nov 2026 00:01:59 +0530 Subject: [PATCH 4/4] test(pyarrow): cover NaN count accumulation and metrics mode --- tests/io/test_format_writers.py | 59 +++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/io/test_format_writers.py b/tests/io/test_format_writers.py index 86ae65156b..cf6764b1c2 100644 --- a/tests/io/test_format_writers.py +++ b/tests/io/test_format_writers.py @@ -24,10 +24,10 @@ import pytest from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel -from pyiceberg.io.pyarrow import PyArrowFileIO +from pyiceberg.io.pyarrow import ParquetFormatModel, PyArrowFileIO, schema_to_pyarrow from pyiceberg.manifest import FileFormat from pyiceberg.schema import Schema -from pyiceberg.types import LongType, NestedField +from pyiceberg.types import DoubleType, FloatType, LongType, NestedField @pytest.fixture(params=FileFormatFactory.available_formats(), ids=lambda f: f.name.lower()) @@ -138,3 +138,58 @@ def test_parquet_format_model_adds_field_id_metadata() -> None: metadata_no_ids: dict[bytes, bytes] = {} ParquetFormatModel().add_field_metadata(field, metadata_no_ids, include_field_ids=False) assert metadata_no_ids == {} + + +def test_parquet_writer_accumulates_nan_value_counts(tmp_path: Path) -> None: + """NaN counts from multiple write() calls are added together.""" + schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer(PyArrowFileIO().new_output(file_path), schema, {}) + + writer.write(pa.Table.from_arrays([pa.array([float("nan"), float("nan")], type=pa.float64())], schema=arrow_schema)) + writer.write(pa.Table.from_arrays([pa.array([1.0, None], type=pa.float64())], schema=arrow_schema)) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {1: 2} + + +def test_parquet_writer_skips_nan_counts_for_none_metrics_mode(tmp_path: Path) -> None: + """A column with metrics mode none does not report a NaN count.""" + schema = Schema( + NestedField(field_id=1, name="f", field_type=FloatType(), required=False), + NestedField(field_id=2, name="d", field_type=DoubleType(), required=False), + ) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer( + PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.column.f": "none"} + ) + + writer.write( + pa.Table.from_arrays( + [ + pa.array([float("nan"), 1.0], type=pa.float32()), + pa.array([float("nan"), 1.0], type=pa.float64()), + ], + schema=arrow_schema, + ) + ) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {2: 1} + + +def test_parquet_writer_default_none_metrics_mode_reports_no_nan_counts(tmp_path: Path) -> None: + """With metrics mode none as the default, no column reports a NaN count.""" + schema = Schema(NestedField(field_id=1, name="d", field_type=DoubleType(), required=False)) + arrow_schema = schema_to_pyarrow(schema, include_field_ids=True) + file_path = str(tmp_path / "test.parquet") + writer = ParquetFormatModel().create_writer( + PyArrowFileIO().new_output(file_path), schema, {"write.metadata.metrics.default": "none"} + ) + + writer.write(pa.Table.from_arrays([pa.array([float("nan"), 1.0], type=pa.float64())], schema=arrow_schema)) + statistics = writer.close() + + assert dict(statistics.nan_value_counts) == {}