Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions pyiceberg/table/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -487,6 +487,16 @@ def construct_partition_specs(cls, data: dict[str, Any]) -> dict[str, Any]:
else:
data[PARTITION_SPECS] = [{"field-id": 0, "fields": ()}]

# In v1 the partition field ids were not tracked, but assigned sequentially from 1000 in each spec
for spec in data[PARTITION_SPECS]:
fields = spec[FIELDS]
missing = sum(1 for field in fields if field.get(FIELD_ID) is None)
if missing == len(fields):
for pos, field in enumerate(fields):
field[FIELD_ID] = PARTITION_FIELD_ID_START + pos
elif missing:
raise ValueError(f"Cannot parse spec with missing field IDs: {missing} missing of {len(fields)} fields.")

data[LAST_PARTITION_ID] = max(
[field.get(FIELD_ID) for spec in data[PARTITION_SPECS] for field in spec[FIELDS]],
default=PARTITION_FIELD_ID_START - 1,
Expand Down
50 changes: 50 additions & 0 deletions tests/table/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,56 @@ def test_migrate_v1_partition_specs(example_table_metadata_v1: dict[str, Any]) -
]


def test_v1_partition_fields_without_field_id(example_table_metadata_v1: dict[str, Any]) -> None:
metadata = copy(example_table_metadata_v1)
metadata["partition-spec"] = [
{"name": "x", "transform": "identity", "source-id": 1},
{"name": "y", "transform": "identity", "source-id": 2},
]
metadata.pop("partition-specs", None)

table_metadata = TableMetadataV1(**metadata)

assert table_metadata.partition_specs == [
PartitionSpec(
PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"),
PartitionField(source_id=2, field_id=1001, transform=IdentityTransform(), name="y"),
)
]
assert table_metadata.last_partition_id == 1001

metadata["partition-specs"] = [
{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1}]},
{
"spec-id": 1,
"fields": [
{"name": "x", "transform": "identity", "source-id": 1},
{"name": "y", "transform": "identity", "source-id": 2},
],
},
]

table_metadata = TableMetadataV1(**metadata)

assert [[field.field_id for field in spec.fields] for spec in table_metadata.partition_specs] == [[1000], [1000, 1001]]
assert table_metadata.last_partition_id == 1001

# A spec that mixes fields with and without ids is rejected, as in Java
for fields in (
[
{"name": "x", "transform": "identity", "source-id": 1},
{"name": "y", "transform": "identity", "source-id": 2, "field-id": 1000},
],
[
{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000},
{"name": "y", "transform": "identity", "source-id": 2},
],
):
metadata["partition-specs"] = [{"spec-id": 0, "fields": fields}]
with pytest.raises(PydanticValidationError, match="Cannot parse spec with missing field IDs: 1 missing of 2 fields"):
TableMetadataV1(**metadata)


def test_new_table_metadata_with_explicit_v1_format() -> None:
schema = Schema(
NestedField(field_id=10, name="foo", field_type=StringType(), required=False),
Expand Down
Loading