diff --git a/pyiceberg/table/inspect.py b/pyiceberg/table/inspect.py index d51079fc9d..2f65447e90 100644 --- a/pyiceberg/table/inspect.py +++ b/pyiceberg/table/inspect.py @@ -195,6 +195,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: partition_record = self.tbl.metadata.specs_struct() pa_record_struct = schema_to_pyarrow(partition_record) + id_to_name = {field.field_id: field.name for field in partition_record.fields} entries_schema = pa.schema( [ @@ -255,7 +256,7 @@ def _readable_metrics_struct(bound_type: PrimitiveType) -> pa.StructType: partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] + id_to_name.get(field.field_id, field.name): partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[manifest.partition_spec_id].fields) } @@ -386,6 +387,7 @@ def partitions( spec_ids = {manifest.partition_spec_id for manifest in snapshot.manifests(self.tbl.io, self.tbl.encryption_manager)} partition_record = self.tbl.metadata.specs_struct(spec_ids=spec_ids) has_partitions = len(partition_record.fields) > 0 + id_to_name = {field.field_id: field.name for field in partition_record.fields} if has_partitions: pa_record_struct = schema_to_pyarrow(partition_record) @@ -407,13 +409,14 @@ def partitions( encryption_manager=self.tbl.encryption_manager, ) - partitions_map: dict[tuple[str, Any], Any] = {} + partitions_map: dict[tuple[int, Any], Any] = {} snapshots_by_id = self._get_snapshots_by_id() for entry in itertools.chain.from_iterable(scan._plan_manifest_entries()): partition = entry.data_file.partition partition_record_dict = { - field.name: partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) + id_to_name.get(field.field_id, field.name): partition[pos] + for pos, field in enumerate(self.tbl.metadata.specs()[entry.data_file.spec_id].fields) } entry_snapshot = snapshots_by_id.get(entry.snapshot_id) if entry.snapshot_id is not None else None self._update_partitions_map_from_manifest_entry( @@ -427,7 +430,7 @@ def partitions( def _update_partitions_map_from_manifest_entry( self, - partitions_map: dict[tuple[str, Any], Any], + partitions_map: dict[tuple[int, Any], Any], file: DataFile, partition_record_dict: dict[str, Any], snapshot: Snapshot | None, @@ -444,7 +447,7 @@ def _update_partitions_map_from_manifest_entry( partition_record_dict: Maps partition field names to their values. snapshot: The owning snapshot, used to track last-updated timestamps. """ - partition_record_key = _convert_to_hashable_type(partition_record_dict) + partition_record_key = (file.spec_id, _convert_to_hashable_type(partition_record_dict)) if partition_record_key not in partitions_map: partitions_map[partition_record_key] = { "partition": partition_record_dict, @@ -737,6 +740,7 @@ def _get_files_from_manifest( files: list[dict[str, Any]] = [] schema = self.tbl.metadata.schema() + id_to_name = {field.field_id: field.name for field in self.tbl.metadata.specs_struct().fields} io = self.tbl.io for manifest_entry in manifest_list.fetch_manifest_entry(io): @@ -762,7 +766,7 @@ def _get_files_from_manifest( } partition = data_file.partition partition_record_dict = { - field.name: partition[pos] + id_to_name.get(field.field_id, field.name): partition[pos] for pos, field in enumerate(self.tbl.metadata.specs()[manifest_list.partition_spec_id].fields) } files.append( diff --git a/tests/table/test_inspect.py b/tests/table/test_inspect.py index 306579754a..7a8095525e 100644 --- a/tests/table/test_inspect.py +++ b/tests/table/test_inspect.py @@ -88,7 +88,7 @@ def test_partitions_last_updated_uses_latest_snapshot_regardless_of_order(newest data_file.spec_id = 0 inspect = InspectTable.__new__(InspectTable) - partitions_map: dict[tuple[str, Any], Any] = {} + partitions_map: dict[tuple[int, Any], Any] = {} for snapshot in [newer, older] if newest_first else [older, newer]: inspect._update_partitions_map_from_manifest_entry(partitions_map, data_file, {"part": "a"}, snapshot) @@ -138,3 +138,59 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None: partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0] assert partition_summaries[0]["lower_bound"] == str(values[0]), name assert partition_summaries[0]["upper_bound"] == str(values[-1]), name + + +def test_inspect_partitions_and_files_after_rename_field(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "p", StringType(), required=False), + ) + spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p")) + tbl = catalog.create_table("default.renamed_partition_col", schema, partition_spec=spec) + tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]})) + + with tbl.update_spec() as spec_update: + spec_update.rename_field("p", "p_renamed") + tbl.append(pa.table({"id": [3], "p": ["c"]})) + + partitions = tbl.inspect.partitions().to_pydict() + assert sorted(part["p_renamed"] for part in partitions["partition"]) == ["a", "b", "c"] + assert set(partitions["spec_id"]) == {0, 1} + + for files_table in [ + tbl.inspect.files(), + tbl.inspect.data_files(), + tbl.inspect.all_files(), + tbl.inspect.all_data_files(), + ]: + assert sorted(part["p_renamed"] for part in files_table.to_pydict()["partition"]) == ["a", "b", "c"] + + entries = tbl.inspect.entries().to_pydict() + assert sorted(df["partition"]["p_renamed"] for df in entries["data_file"]) == ["a", "b", "c"] + + +def test_inspect_files_after_v1_partition_field_readd(catalog: InMemoryCatalog) -> None: + schema = Schema( + NestedField(1, "id", LongType(), required=False), + NestedField(2, "p", StringType(), required=False), + ) + spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p")) + tbl = catalog.create_table("default.v1_partition_readd", schema, partition_spec=spec, properties={"format-version": "1"}) + tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"]})) + + with tbl.update_spec() as spec_update: + spec_update.remove_field("p") + with tbl.update_spec() as spec_update: + spec_update.add_identity("p") + + # In v1, the voided field is renamed to p_1000 and the new identity field is assigned id 1001 with name p. + # Pre-existing data files from spec 0 belong to field 1000 (p_1000), not the newly added field 1001 (p). + for files_table in [tbl.inspect.files(), tbl.inspect.all_files()]: + partitions = files_table.to_pydict()["partition"] + assert sorted(part["p_1000"] for part in partitions) == ["a", "b"] + assert all(part["p"] is None for part in partitions) + + entries = tbl.inspect.entries().to_pydict() + entry_partitions = [df["partition"] for df in entries["data_file"]] + assert sorted(part["p_1000"] for part in entry_partitions) == ["a", "b"] + assert all(part["p"] is None for part in entry_partitions)