|
28 | 28 | timezone, |
29 | 29 | ) |
30 | 30 | from decimal import Decimal |
| 31 | +from types import SimpleNamespace |
31 | 32 | from typing import ( |
32 | 33 | Any, |
33 | 34 | ) |
|
43 | 44 | STRUCT_INT32, |
44 | 45 | STRUCT_INT64, |
45 | 46 | ) |
| 47 | +from pyiceberg.expressions import IsNull |
| 48 | +from pyiceberg.expressions.visitors import _InclusiveMetricsEvaluator |
46 | 49 | from pyiceberg.io.pyarrow import ( |
47 | 50 | MetricModeTypes, |
48 | 51 | MetricsMode, |
|
53 | 56 | parquet_path_to_id_mapping, |
54 | 57 | schema_to_pyarrow, |
55 | 58 | ) |
56 | | -from pyiceberg.manifest import DataFile |
| 59 | +from pyiceberg.manifest import DataFile, FileFormat |
57 | 60 | from pyiceberg.schema import Schema, pre_order_visit |
58 | 61 | from pyiceberg.table.metadata import ( |
59 | 62 | TableMetadata, |
60 | 63 | TableMetadataUtil, |
61 | 64 | TableMetadataV1, |
62 | 65 | TableMetadataV2, |
63 | 66 | ) |
| 67 | +from pyiceberg.typedef import Record |
64 | 68 | from pyiceberg.types import ( |
65 | 69 | BooleanType, |
66 | 70 | FloatType, |
67 | 71 | IntegerType, |
| 72 | + NestedField, |
68 | 73 | StringType, |
69 | 74 | ) |
70 | 75 | from pyiceberg.utils.datetime import date_to_days, datetime_to_micros, time_to_micros |
@@ -262,6 +267,64 @@ def test_null_and_nan_counts() -> None: |
262 | 267 | # assert datafile.nan_value_counts[3] == 0 |
263 | 268 |
|
264 | 269 |
|
| 270 | +@pytest.mark.parametrize("unknown_row_group", [0, 1]) |
| 271 | +def test_unknown_row_group_null_count_is_omitted(unknown_row_group: int) -> None: |
| 272 | + schema = Schema(NestedField(1, "value", StringType())) |
| 273 | + arrow_schema = schema_to_pyarrow(schema) |
| 274 | + table = pa.table({"value": ["present", None]}, schema=arrow_schema) |
| 275 | + metadata_collector: list[Any] = [] |
| 276 | + |
| 277 | + with pa.BufferOutputStream() as output: |
| 278 | + with pq.ParquetWriter(output, table.schema, metadata_collector=metadata_collector) as writer: |
| 279 | + writer.write_table(table, row_group_size=1) |
| 280 | + |
| 281 | + metadata = metadata_collector[0] |
| 282 | + assert metadata.num_row_groups == 2 |
| 283 | + assert metadata.row_group(0).column(0).statistics.null_count == 0 |
| 284 | + assert metadata.row_group(1).column(0).statistics.null_count == 1 |
| 285 | + assert table.column("value").null_count == 1 |
| 286 | + |
| 287 | + row_groups: list[Any] = [] |
| 288 | + for index in range(metadata.num_row_groups): |
| 289 | + column = metadata.row_group(index).column(0) |
| 290 | + column_metadata = SimpleNamespace( |
| 291 | + data_page_offset=column.data_page_offset, |
| 292 | + dictionary_page_offset=column.dictionary_page_offset, |
| 293 | + has_dictionary_page=column.has_dictionary_page, |
| 294 | + path_in_schema=column.path_in_schema, |
| 295 | + total_compressed_size=column.total_compressed_size, |
| 296 | + num_values=column.num_values, |
| 297 | + is_stats_set=column.is_stats_set, |
| 298 | + statistics=SimpleNamespace( |
| 299 | + has_null_count=index != unknown_row_group, |
| 300 | + null_count=column.statistics.null_count, |
| 301 | + ), |
| 302 | + ) |
| 303 | + row_groups.append(SimpleNamespace(column=lambda _index, column=column_metadata: column)) |
| 304 | + metadata_view = SimpleNamespace( |
| 305 | + num_row_groups=metadata.num_row_groups, |
| 306 | + num_columns=metadata.num_columns, |
| 307 | + num_rows=metadata.num_rows, |
| 308 | + row_group=lambda index: row_groups[index], |
| 309 | + ) |
| 310 | + |
| 311 | + statistics = data_file_statistics_from_parquet_metadata( |
| 312 | + parquet_metadata=metadata_view, |
| 313 | + stats_columns=compute_statistics_plan(schema, {"write.metadata.metrics.default": "counts"}), |
| 314 | + parquet_column_mapping=parquet_path_to_id_mapping(schema), |
| 315 | + ) |
| 316 | + assert statistics.null_value_counts == {} |
| 317 | + |
| 318 | + data_file = DataFile.from_args( |
| 319 | + file_path="file.parquet", |
| 320 | + file_format=FileFormat.PARQUET, |
| 321 | + partition=Record(), |
| 322 | + file_size_in_bytes=metadata.serialized_size, |
| 323 | + **statistics.to_serialized_dict(), |
| 324 | + ) |
| 325 | + assert _InclusiveMetricsEvaluator(schema, IsNull("value")).eval(data_file) |
| 326 | + |
| 327 | + |
265 | 328 | def test_bounds() -> None: |
266 | 329 | metadata, table_metadata = construct_test_table() |
267 | 330 |
|
|
0 commit comments