Skip to content

Commit 33b4ce5

Browse files
iceberg-kittenMinh Vu
andauthored
fix(pyarrow): omit incomplete parquet null counts (#4385)
Co-authored-by: Minh Vu <vuhoangminh97@gmail.com>
1 parent fc4d884 commit 33b4ce5

2 files changed

Lines changed: 70 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2749,6 +2749,7 @@ def data_file_statistics_from_parquet_metadata(
27492749
col_aggs = {}
27502750

27512751
invalidate_col: set[int] = set()
2752+
invalidate_null_counts: set[int] = set()
27522753
without_bounds: set[int] = set()
27532754
for r in range(parquet_metadata.num_row_groups):
27542755
# References:
@@ -2785,6 +2786,8 @@ def data_file_statistics_from_parquet_metadata(
27852786

27862787
if statistics.has_null_count:
27872788
null_value_counts[field_id] = null_value_counts.get(field_id, 0) + statistics.null_count
2789+
else:
2790+
invalidate_null_counts.add(field_id)
27882791

27892792
if stats_col.mode == MetricsMode(MetricModeTypes.COUNTS):
27902793
continue
@@ -2831,6 +2834,9 @@ def data_file_statistics_from_parquet_metadata(
28312834
col_aggs.pop(field_id, None)
28322835
null_value_counts.pop(field_id, None)
28332836

2837+
for field_id in invalidate_null_counts:
2838+
null_value_counts.pop(field_id, None)
2839+
28342840
for field_id in without_bounds:
28352841
col_aggs.pop(field_id, None)
28362842

‎tests/io/test_pyarrow_stats.py‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
timezone,
2929
)
3030
from decimal import Decimal
31+
from types import SimpleNamespace
3132
from typing import (
3233
Any,
3334
)
@@ -43,6 +44,8 @@
4344
STRUCT_INT32,
4445
STRUCT_INT64,
4546
)
47+
from pyiceberg.expressions import IsNull
48+
from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator
4649
from pyiceberg.io.pyarrow import (
4750
MetricModeTypes,
4851
MetricsMode,
@@ -53,18 +56,20 @@
5356
parquet_path_to_id_mapping,
5457
schema_to_pyarrow,
5558
)
56-
from pyiceberg.manifest import DataFile
59+
from pyiceberg.manifest import DataFile, FileFormat
5760
from pyiceberg.schema import Schema, pre_order_visit
5861
from pyiceberg.table.metadata import (
5962
TableMetadata,
6063
TableMetadataUtil,
6164
TableMetadataV1,
6265
TableMetadataV2,
6366
)
67+
from pyiceberg.typedef import Record
6468
from pyiceberg.types import (
6569
BooleanType,
6670
FloatType,
6771
IntegerType,
72+
NestedField,
6873
StringType,
6974
)
7075
from pyiceberg.utils.datetime import date_to_days, datetime_to_micros, time_to_micros
@@ -262,6 +267,64 @@ def test_null_and_nan_counts() -> None:
262267
# assert datafile.nan_value_counts[3] == 0
263268

264269

270+
@pytest.mark.parametrize("unknown_row_group", [0, 1])
271+
def test_unknown_row_group_null_count_is_omitted(unknown_row_group: int) -> None:
272+
schema = Schema(NestedField(1, "value", StringType()))
273+
arrow_schema = schema_to_pyarrow(schema)
274+
table = pa.table({"value": ["present", None]}, schema=arrow_schema)
275+
metadata_collector: list[Any] = []
276+
277+
with pa.BufferOutputStream() as output:
278+
with pq.ParquetWriter(output, table.schema, metadata_collector=metadata_collector) as writer:
279+
writer.write_table(table, row_group_size=1)
280+
281+
metadata = metadata_collector[0]
282+
assert metadata.num_row_groups == 2
283+
assert metadata.row_group(0).column(0).statistics.null_count == 0
284+
assert metadata.row_group(1).column(0).statistics.null_count == 1
285+
assert table.column("value").null_count == 1
286+
287+
row_groups: list[Any] = []
288+
for index in range(metadata.num_row_groups):
289+
column = metadata.row_group(index).column(0)
290+
column_metadata = SimpleNamespace(
291+
data_page_offset=column.data_page_offset,
292+
dictionary_page_offset=column.dictionary_page_offset,
293+
has_dictionary_page=column.has_dictionary_page,
294+
path_in_schema=column.path_in_schema,
295+
total_compressed_size=column.total_compressed_size,
296+
num_values=column.num_values,
297+
is_stats_set=column.is_stats_set,
298+
statistics=SimpleNamespace(
299+
has_null_count=index != unknown_row_group,
300+
null_count=column.statistics.null_count,
301+
),
302+
)
303+
row_groups.append(SimpleNamespace(column=lambda _index, column=column_metadata: column))
304+
metadata_view = SimpleNamespace(
305+
num_row_groups=metadata.num_row_groups,
306+
num_columns=metadata.num_columns,
307+
num_rows=metadata.num_rows,
308+
row_group=lambda index: row_groups[index],
309+
)
310+
311+
statistics = data_file_statistics_from_parquet_metadata(
312+
parquet_metadata=metadata_view,
313+
stats_columns=compute_statistics_plan(schema, {"write.metadata.metrics.default": "counts"}),
314+
parquet_column_mapping=parquet_path_to_id_mapping(schema),
315+
)
316+
assert statistics.null_value_counts == {}
317+
318+
data_file = DataFile.from_args(
319+
file_path="file.parquet",
320+
file_format=FileFormat.PARQUET,
321+
partition=Record(),
322+
file_size_in_bytes=metadata.serialized_size,
323+
**statistics.to_serialized_dict(),
324+
)
325+
assert _InclusiveMetricsEvaluator(schema, IsNull("value")).eval(data_file)
326+
327+
265328
def test_bounds() -> None:
266329
metadata, table_metadata = construct_test_table()
267330

0 commit comments

Comments
 (0)