Skip to content

Commit 2b0925a

Browse files
committed
fix(inspect): resolve dropped partition source columns in specs_struct
When a partition column is removed and subsequently dropped from the table schema, historical partition specs still reference the dropped source_id. specs_struct() previously only looked up field IDs in the current schema, causing inspect metadata tables to fail with ValueError and manifests partition summaries to lose bound values. This updates specs_struct() to fall back to historical schemas when resolving source column types, and updates inspect manifest generation to look up partition types from specs_struct().
1 parent c4cd513 commit 2b0925a

4 files changed

Lines changed: 143 additions & 6 deletions

File tree

‎pyiceberg/table/inspect.py‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
from pyiceberg.manifest import DataFile, DataFileContent, ManifestContent, ManifestFile, PartitionFieldSummary
2727
from pyiceberg.partitioning import PartitionSpec
2828
from pyiceberg.table.snapshots import Snapshot, ancestors_of
29-
from pyiceberg.types import PrimitiveType
29+
from pyiceberg.types import IcebergType, PrimitiveType
3030
from pyiceberg.utils.concurrent import ExecutorFactory
3131
from pyiceberg.utils.singleton import _convert_to_hashable_type
3232

@@ -553,13 +553,23 @@ def _generate_manifests_table(self, snapshot: Snapshot | None, is_all_manifests_
553553
"""
554554
import pyarrow as pa
555555

556+
partition_types_by_spec_id: dict[int, dict[int, IcebergType]] = {}
557+
556558
def _partition_summaries_to_rows(
557559
spec: PartitionSpec, partition_summaries: list[PartitionFieldSummary]
558560
) -> list[dict[str, Any]]:
561+
partition_field_types = partition_types_by_spec_id.get(spec.spec_id)
562+
if partition_field_types is None:
563+
partition_field_types = {
564+
struct_field.field_id: struct_field.field_type
565+
for struct_field in self.tbl.metadata.specs_struct([spec.spec_id]).fields
566+
}
567+
partition_types_by_spec_id[spec.spec_id] = partition_field_types
568+
559569
rows = []
560570
for i, field_summary in enumerate(partition_summaries):
561571
field = spec.fields[i]
562-
partition_field_type = spec.partition_type(self.tbl.schema()).fields[i].field_type
572+
partition_field_type = partition_field_types[field.field_id]
563573
lower_bound = (
564574
(
565575
field.transform.to_human_string(

‎pyiceberg/table/metadata.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@
4545
IcebergRootModel,
4646
Properties,
4747
)
48-
from pyiceberg.types import NestedField, StructType, transform_dict_value_to_str
48+
from pyiceberg.types import IcebergType, NestedField, StructType, UnknownType, transform_dict_value_to_str
4949
from pyiceberg.utils.config import Config
5050
from pyiceberg.utils.datetime import datetime_to_millis
5151

@@ -298,6 +298,16 @@ def specs(self) -> dict[int, PartitionSpec]:
298298
"""Return a dict the partition specs this table."""
299299
return {spec.spec_id: spec for spec in self.partition_specs}
300300

301+
def _resolve_field_type(self, field_id: int) -> IcebergType:
302+
"""Resolve a field's type from the current schema or historical schemas."""
303+
if field := self.schema()._lazy_id_to_field.get(field_id):
304+
return field.field_type
305+
# Historical partition specs may reference columns dropped in later schema evolutions.
306+
for schema in reversed(self.schemas):
307+
if field := schema._lazy_id_to_field.get(field_id):
308+
return field.field_type
309+
return UnknownType()
310+
301311
def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType:
302312
"""Produce a struct of the combined PartitionSpecs.
303313
@@ -316,13 +326,11 @@ def specs_struct(self, spec_ids: Iterable[int] | None = None) -> StructType:
316326
# Collect all the fields
317327
struct_fields = {field.field_id: field for spec in selected_specs for field in spec.fields}
318328

319-
schema = self.schema()
320-
321329
nested_fields = []
322330
# Sort them by field_id in order to get a deterministic output
323331
for field_id in sorted(struct_fields):
324332
field = struct_fields[field_id]
325-
source_type = schema.find_type(field.source_id)
333+
source_type = self._resolve_field_type(field.source_id)
326334
result_type = field.transform.result_type(source_type)
327335
nested_fields.append(NestedField(field_id=field.field_id, name=field.name, type=result_type, required=False))
328336

‎tests/table/test_inspect.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,3 +138,72 @@ def test_inspect_bounds_after_type_promotion(catalog: InMemoryCatalog) -> None:
138138
partition_summaries = tbl.inspect.manifests().to_pydict()["partition_summaries"][0]
139139
assert partition_summaries[0]["lower_bound"] == str(values[0]), name
140140
assert partition_summaries[0]["upper_bound"] == str(values[-1]), name
141+
142+
143+
def test_inspect_after_dropping_old_partition_column(catalog: InMemoryCatalog) -> None:
144+
schema = Schema(
145+
NestedField(1, "id", LongType(), required=False),
146+
NestedField(2, "p", StringType(), required=False),
147+
NestedField(3, "q", StringType(), required=False),
148+
)
149+
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"))
150+
tbl = catalog.create_table("default.dropped_partition_col", schema, partition_spec=spec)
151+
tbl.append(pa.table({"id": [1, 2], "p": ["a", "b"], "q": ["x", "y"]}))
152+
153+
with tbl.update_spec() as spec_update:
154+
spec_update.remove_field("p")
155+
with tbl.update_schema() as schema_update:
156+
schema_update.delete_column("p")
157+
158+
assert tbl.inspect.partitions().to_pydict()["partition"] == [{"p": "a"}, {"p": "b"}]
159+
for table_view in [
160+
tbl.inspect.entries(),
161+
tbl.inspect.files(),
162+
tbl.inspect.data_files(),
163+
tbl.inspect.all_files(),
164+
tbl.inspect.all_data_files(),
165+
]:
166+
assert table_view.num_rows == 2
167+
assert tbl.inspect.delete_files().num_rows == 0
168+
assert tbl.inspect.all_delete_files().num_rows == 0
169+
170+
for manifest_table in [tbl.inspect.manifests(), tbl.inspect.all_manifests()]:
171+
assert manifest_table.to_pydict()["partition_summaries"][0] == [
172+
{"contains_null": False, "contains_nan": False, "lower_bound": "a", "upper_bound": "b"}
173+
]
174+
175+
# Appending under an unpartitioned spec and a newly evolved partition column also stays readable
176+
tbl.append(pa.table({"id": [3], "q": ["z"]}))
177+
with tbl.update_spec() as spec_update:
178+
spec_update.add_identity("q")
179+
tbl.append(pa.table({"id": [4], "q": ["w"]}))
180+
181+
assert sorted(
182+
[(row["p"], row["q"]) for row in tbl.inspect.partitions().to_pydict()["partition"]],
183+
key=lambda item: (item[0] or "", item[1] or ""),
184+
) == [(None, None), (None, "w"), ("a", None), ("b", None)]
185+
assert tbl.inspect.entries().num_rows == 4
186+
assert tbl.inspect.files().num_rows == 4
187+
assert tbl.inspect.all_files().num_rows == 4
188+
189+
190+
def test_inspect_after_promoting_then_dropping_partition_column(catalog: InMemoryCatalog) -> None:
191+
schema = Schema(
192+
NestedField(1, "id", LongType(), required=False),
193+
NestedField(2, "v", IntegerType(), required=False),
194+
)
195+
spec = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "v"))
196+
tbl = catalog.create_table("default.promoted_then_dropped", schema, partition_spec=spec)
197+
tbl.append(pa.table({"id": pa.array([1, 2], pa.int64()), "v": pa.array([10, 20], pa.int32())}))
198+
199+
with tbl.update_schema() as schema_update:
200+
schema_update.update_column("v", field_type=LongType())
201+
with tbl.update_spec() as spec_update:
202+
spec_update.remove_field("v")
203+
with tbl.update_schema() as schema_update:
204+
schema_update.delete_column("v")
205+
206+
assert sorted(row["v"] for row in tbl.inspect.partitions().to_pydict()["partition"]) == [10, 20]
207+
assert tbl.inspect.manifests().to_pydict()["partition_summaries"][0] == [
208+
{"contains_null": False, "contains_nan": False, "lower_bound": "10", "upper_bound": "20"}
209+
]

‎tests/table/test_metadata.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -992,3 +992,53 @@ def test_v3_metadata_with_encryption_keys_serializes_them(example_table_metadata
992992
assert table_metadata.model_dump(mode="json")["encryption-keys"] == [
993993
{"key-id": "a", "encrypted-key-metadata": "a2V5", "properties": {}}
994994
]
995+
996+
997+
def test_specs_struct_resolves_dropped_partition_source_from_historical_schemas() -> None:
998+
initial_schema = Schema(
999+
NestedField(1, "id", LongType()),
1000+
NestedField(2, "p", StringType()),
1001+
NestedField(3, "v", IntegerType()),
1002+
schema_id=0,
1003+
)
1004+
promoted_schema = Schema(
1005+
NestedField(1, "id", LongType()),
1006+
NestedField(2, "p", StringType()),
1007+
NestedField(3, "v", LongType()),
1008+
schema_id=1,
1009+
)
1010+
dropped_schema = Schema(NestedField(1, "id", LongType()), schema_id=2)
1011+
1012+
spec_p = PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0)
1013+
spec_v = PartitionSpec(PartitionField(3, 1001, IdentityTransform(), "v"), spec_id=1)
1014+
1015+
table_metadata = TableMetadataV2(
1016+
location="s3://bucket/test/location",
1017+
schemas=[initial_schema, promoted_schema, dropped_schema],
1018+
current_schema_id=2,
1019+
partition_specs=[spec_p, spec_v, PartitionSpec(spec_id=2)],
1020+
default_spec_id=2,
1021+
last_column_id=3,
1022+
last_partition_id=1001,
1023+
)
1024+
1025+
assert table_metadata.specs_struct() == StructType(
1026+
NestedField(1000, "p", StringType(), required=False),
1027+
NestedField(1001, "v", LongType(), required=False),
1028+
)
1029+
1030+
1031+
def test_specs_struct_falls_back_to_unknown_when_source_missing_from_all_schemas() -> None:
1032+
table_metadata = TableMetadataV2(
1033+
location="s3://bucket/test/location",
1034+
schemas=[Schema(NestedField(1, "id", LongType()), schema_id=0)],
1035+
current_schema_id=0,
1036+
partition_specs=[PartitionSpec(PartitionField(2, 1000, IdentityTransform(), "p"), spec_id=0)],
1037+
default_spec_id=0,
1038+
last_column_id=2,
1039+
last_partition_id=1000,
1040+
)
1041+
1042+
assert table_metadata.specs_struct() == StructType(
1043+
NestedField(1000, "p", UnknownType(), required=False),
1044+
)

0 commit comments

Comments
 (0)