Skip to content

Commit d8cf7b2

Browse files
committed
fix(inspect): map partition record by field id to support renamed partition fields
When a partition field is renamed (via update_spec().rename_field() or via v1 auto-renaming voided fields on re-add), specs_struct() adopts the latest field names per field_id. Previously, inspect.py constructed partition dicts using the historical spec's field names directly, causing PyArrow conversion to silently drop pre-rename partition values as None (or misattribute them to newly added fields sharing the old name). This updates InspectTable.entries(), partitions(), and _get_files_from_manifest() to map partition field values by field_id using specs_struct() names, and keys partitions_map by (spec_id, partition_record_dict) to track partition stats per specification layout.
1 parent c4cd513 commit d8cf7b2

2 files changed

Lines changed: 68 additions & 6 deletions

File tree

‎pyiceberg/table/inspect.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
195195

196196
partition_record = self.tbl.metadata.specs_struct()
197197
pa_record_struct = schema_to_pyarrow(partition_record)
198+
id_to_name = {field.field_id: field.name for field in partition_record.fields}
198199

199200
entries_schema = pa.schema(
200201
[
@@ -255,7 +256,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType:
255256

256257
partition = entry.data_file.partition
257258
partition_record_dict = {
258-
field.name: partition[pos]
259+
id_to_name.get(field.field_id, field.name): partition[pos]
259260
for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields)
260261
}
261262

@@ -386,6 +387,7 @@ def partitions(
386387
spec_ids = {manifest.partition_spec_id for manifest in snapshot.manifests(self.tbl.io, self.tbl.encryption_manager)}
387388
partition_record = self.tbl.metadata.specs_struct(spec_ids=spec_ids)
388389
has_partitions = len(partition_record.fields) > 0
390+
id_to_name = {field.field_id: field.name for field in partition_record.fields}
389391

390392
if has_partitions:
391393
pa_record_struct = schema_to_pyarrow(partition_record)
@@ -407,13 +409,14 @@ def partitions(
407409
encryption_manager=self.tbl.encryption_manager,
408410
)
409411

410-
partitions_map: dict[tuple[str, Any], Any] = {}
412+
partitions_map: dict[tuple[int, Any], Any] = {}
411413
snapshots_by_id = self._get_snapshots_by_id()
412414

413415
for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()):
414416
partition = entry.data_file.partition
415417
partition_record_dict = {
416-
field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
418+
id_to_name.get(field.field_id, field.name): partition[pos]
419+
for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields)
417420
}
418421
entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None
419422
self._update_partitions_map_from_manifest_entry(
@@ -427,7 +430,7 @@ def partitions(
427430

428431
def _update_partitions_map_from_manifest_entry(
429432
self,
430-
partitions_map: dict[tuple[str, Any], Any],
433+
partitions_map: dict[tuple[int, Any], Any],
431434
file: DataFile,
432435
partition_record_dict: dict[str, Any],
433436
snapshot: Snapshot | None,
@@ -444,7 +447,7 @@ def _update_partitions_map_from_manifest_entry(
444447
partition_record_dict: Maps partition field names to their values.
445448
snapshot: The owning snapshot, used to track last-updated timestamps.
446449
"""
447-
partition_record_key = _convert_to_hashable_type(partition_record_dict)
450+
partition_record_key = (file.spec_id, _convert_to_hashable_type(partition_record_dict))
448451
if partition_record_key not in partitions_map:
449452
partitions_map[partition_record_key] = {
450453
"partition": partition_record_dict,
@@ -737,6 +740,7 @@ def _get_files_from_manifest(
737740

738741
files: list[dict[str, Any]] = []
739742
schema = self.tbl.metadata.schema()
743+
id_to_name = {field.field_id: field.name for field in self.tbl.metadata.specs_struct().fields}
740744
io = self.tbl.io
741745

742746
for manifest_entry in manifest_list.fetch_manifest_entry(io):
@@ -762,7 +766,7 @@ def _get_files_from_manifest(
762766
}
763767
partition = data_file.partition
764768
partition_record_dict = {
765-
field.name: partition[pos]
769+
id_to_name.get(field.field_id, field.name): partition[pos]
766770
for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields)
767771
}
768772
files.append(

‎tests/table/test_inspect.py‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,3 +138,61 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None:
138138
partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0]
139139
assert partition_summaries[0]["lower_bound"] == str(values[0]), name
140140
assert partition_summaries[0]["upper_bound"] == str(values[-1]), name
141+
142+
143+
def test_inspect_partitions_and_files_after_rename_field(catalog: InMemoryCatalog) -> None:
144+
schema = Schema(
145+
NestedField(1, "id", LongType(), required=False),
146+
NestedField(2, "p", StringType(), required=False),
147+
)
148+
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
149+
tbl = catalog.create_table("default.renamed_partition_col", schema, partition_spec=spec)
150+
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]}))
151+
152+
with tbl.update_spec() as spec_update:
153+
spec_update.rename_field("p", "p_renamed")
154+
tbl.append(pa.table({"id": [3], "p": ["c"]}))
155+
156+
partitions = tbl.inspect.partitions().to_pydict()
157+
assert sorted(part["p_renamed"] for part in partitions["partition"]) == ["a", "b", "c"]
158+
assert set(partitions["spec_id"]) == {0, 1}
159+
160+
for files_table in [
161+
tbl.inspect.files(),
162+
tbl.inspect.data_files(),
163+
tbl.inspect.all_files(),
164+
tbl.inspect.all_data_files(),
165+
]:
166+
assert sorted(part["p_renamed"] for part in files_table.to_pydict()["partition"]) == ["a", "b", "c"]
167+
168+
entries = tbl.inspect.entries().to_pydict()
169+
assert sorted(df["partition"]["p_renamed"] for df in entries["data_file"]) == ["a", "b", "c"]
170+
171+
172+
def test_inspect_files_after_v1_partition_field_readd(catalog: InMemoryCatalog) -> None:
173+
schema = Schema(
174+
NestedField(1, "id", LongType(), required=False),
175+
NestedField(2, "p", StringType(), required=False),
176+
)
177+
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
178+
tbl = catalog.create_table(
179+
"default.v1_partition_readd", schema, partition_spec=spec, properties={"format-version": "1"}
180+
)
181+
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]}))
182+
183+
with tbl.update_spec() as spec_update:
184+
spec_update.remove_field("p")
185+
with tbl.update_spec() as spec_update:
186+
spec_update.add_identity("p")
187+
188+
# In v1, the voided field is renamed to p_1000 and the new identity field is assigned id 1001 with name p.
189+
# Pre-existing data files from spec 0 belong to field 1000 (p_1000), not the newly added field 1001 (p).
190+
for files_table in [tbl.inspect.files(), tbl.inspect.all_files()]:
191+
partitions = files_table.to_pydict()["partition"]
192+
assert sorted(part["p_1000"] for part in partitions) == ["a", "b"]
193+
assert all(part["p"] is None for part in partitions)
194+
195+
entries = tbl.inspect.entries().to_pydict()
196+
entry_partitions = [df["partition"] for df in entries["data_file"]]
197+
assert sorted(part["p_1000"] for part in entry_partitions) == ["a", "b"]
198+
assert all(part["p"] is None for part in entry_partitions)

0 commit comments

Comments
 (0)