diff --git a/pyiceberg/table/metadata.py b/pyiceberg/table/metadata.py index 9e983667ed..26c205afd6 100644 --- a/pyiceberg/table/metadata.py +++ b/pyiceberg/table/metadata.py @@ -487,6 +487,16 @@ def construct_partition_specs(cls, data: dict[str, Any]) -> dict[str, Any]: else: data[PARTITION_SPECS] = [{"field-id": 0, "fields": ()}] + # In v1 the partition field ids were not tracked, but assigned sequentially from 1000 in each spec + for spec in data[PARTITION_SPECS]: + fields = spec[FIELDS] + missing = sum(1 for field in fields if field.get(FIELD_ID) is None) + if missing == len(fields): + for pos, field in enumerate(fields): + field[FIELD_ID] = PARTITION_FIELD_ID_START + pos + elif missing: + raise ValueError(f"Cannot parse spec with missing field IDs: {missing} missing of {len(fields)} fields.") + data[LAST_PARTITION_ID] = max( [field.get(FIELD_ID) for spec in data[PARTITION_SPECS] for field in spec[FIELDS]], default=PARTITION_FIELD_ID_START - 1, diff --git a/tests/table/test_metadata.py b/tests/table/test_metadata.py index fb20f9726a..a261d7823a 100644 --- a/tests/table/test_metadata.py +++ b/tests/table/test_metadata.py @@ -213,6 +213,56 @@ def test_migrate_v1_partition_specs(example_table_metadata_v1: dict[str, Any]) - ] +def test_v1_partition_fields_without_field_id(example_table_metadata_v1: dict[str, Any]) -> None: + metadata = copy(example_table_metadata_v1) + metadata["partition-spec"] = [ + {"name": "x", "transform": "identity", "source-id": 1}, + {"name": "y", "transform": "identity", "source-id": 2}, + ] + metadata.pop("partition-specs", None) + + table_metadata = TableMetadataV1(**metadata) + + assert table_metadata.partition_specs == [ + PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), + PartitionField(source_id=2, field_id=1001, transform=IdentityTransform(), name="y"), + ) + ] + assert table_metadata.last_partition_id == 1001 + + metadata["partition-specs"] = [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1}]}, + { + "spec-id": 1, + "fields": [ + {"name": "x", "transform": "identity", "source-id": 1}, + {"name": "y", "transform": "identity", "source-id": 2}, + ], + }, + ] + + table_metadata = TableMetadataV1(**metadata) + + assert [[field.field_id for field in spec.fields] for spec in table_metadata.partition_specs] == [[1000], [1000, 1001]] + assert table_metadata.last_partition_id == 1001 + + # A spec that mixes fields with and without ids is rejected, as in Java + for fields in ( + [ + {"name": "x", "transform": "identity", "source-id": 1}, + {"name": "y", "transform": "identity", "source-id": 2, "field-id": 1000}, + ], + [ + {"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}, + {"name": "y", "transform": "identity", "source-id": 2}, + ], + ): + metadata["partition-specs"] = [{"spec-id": 0, "fields": fields}] + with pytest.raises(PydanticValidationError, match="Cannot parse spec with missing field IDs: 1 missing of 2 fields"): + TableMetadataV1(**metadata) + + def test_new_table_metadata_with_explicit_v1_format() -> None: schema = Schema( NestedField(field_id=10, name="foo", field_type=StringType(), required=False),