Skip to content

Commit b84df8d

Browse files
Rahul-cs50kevinjqliuCopilot
authored
Fix inspect.entries() when column_sizes is missing (#4199)
* Fix inspect.entries() for data files without column_sizes column_sizes is optional in a data file, but entries() built it with dict(entry.data_file.column_sizes), which raises TypeError when it is missing. DuckDB writes data files without it. Use the `or {}` that Closes #4186 * Shorten the comment in the missing-metrics test * Set spec_id on the test DataFile The snapshot producer now reads DataFile.spec_id, so the test failed after rebasing onto main. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Drop the Parquet write from the missing-metrics test inspect only reads manifests, so the data file doesn't need to exist. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Kevin Liu <kevin.jq.liu@gmail.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 046e584 commit b84df8d

2 files changed

Lines changed: 30 additions & 2 deletions

File tree

‎pyiceberg/table/inspect.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
272272
"partition": partition_record_dict,
273273
"record_count": entry.data_file.record_count,
274274
"file_size_in_bytes": entry.data_file.file_size_in_bytes,
275-
"column_sizes": dict(entry.data_file.column_sizes),
275+
"column_sizes": dict(entry.data_file.column_sizes or {}),
276276
"value_counts": dict(entry.data_file.value_counts or {}),
277277
"null_value_counts": dict(entry.data_file.null_value_counts or {}),
278278
"nan_value_counts": dict(entry.data_file.nan_value_counts or {}),

‎tests/table/test_inspect.py‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import pytest
2323

2424
from pyiceberg.conversions import to_bytes
25-
from pyiceberg.manifest import DataFile, DataFileContent
25+
from pyiceberg.manifest import DataFile, DataFileContent, FileFormat
2626
from pyiceberg.partitioning import PartitionField, PartitionSpec
2727
from pyiceberg.schema import Schema
2828
from pyiceberg.table.inspect import InspectTable, _readable_bound
@@ -76,6 +76,34 @@ def test_inspect_entries_and_files_render_null_bound(catalog: InMemoryCatalog) -
7676
assert files_metrics["upper_bound"] is None
7777

7878

79+
def test_inspect_entries_and_files_with_data_file_without_metrics(catalog: InMemoryCatalog) -> None:
80+
# column_sizes and the other metric maps are optional in a data file
81+
schema = Schema(NestedField(1, "x", LongType(), required=False))
82+
tbl = catalog.create_table("default.no_metrics", schema)
83+
data_file = DataFile.from_args(
84+
content=DataFileContent.DATA,
85+
file_path=f"{tbl.location()}/data/a.parquet",
86+
file_format=FileFormat.PARQUET,
87+
partition=Record(),
88+
record_count=1,
89+
file_size_in_bytes=1,
90+
spec_id=tbl.metadata.default_spec_id,
91+
)
92+
with tbl.transaction() as tx:
93+
with tx.update_snapshot().fast_append() as fast_append:
94+
fast_append.append_data_file(data_file)
95+
96+
entries = tbl.inspect.entries().to_pylist()
97+
assert len(entries) == 1
98+
for metric in ("column_sizes", "value_counts", "null_value_counts", "nan_value_counts"):
99+
assert entries[0]["data_file"][metric] == []
100+
assert entries[0]["readable_metrics"]["x"]["column_size"] is None
101+
102+
files = tbl.inspect.files().to_pylist()
103+
assert len(files) == 1
104+
assert files[0]["column_sizes"] is None
105+
106+
79107
@pytest.mark.parametrize("newest_first", [False, True])
80108
def test_partitions_last_updated_uses_latest_snapshot_regardless_of_order(newest_first: bool) -> None:
81109
# Manifest entries are visited in manifest order, which is not chronological, so the

0 commit comments

Comments
 (0)