Skip to content

Commit 533dea9

Browse files
authored
Reject duplicate partition field names (#4305)
* Reject duplicate partition field names validate_partition_name returned early when the partition name was not also a schema column, so its empty-name and uniqueness checks only ran for names that matched a column. update_spec could then commit a spec with two fields of the same name (for example through rename_field), after which appends failed and Java cannot load the metadata. create_table did not check uniqueness at all because it passed a fresh set for every field. Run the empty-name and uniqueness checks first, and track the names seen so far in assign_fresh_partition_spec_ids. * Move the empty partition name case to its own test
1 parent f2ca41e commit 533dea9

3 files changed

Lines changed: 45 additions & 5 deletions

File tree

‎pyiceberg/partitioning.py‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -300,6 +300,11 @@ def validate_partition_name(
300300
partition_names: set[str],
301301
) -> None:
302302
"""Validate that a partition field name doesn't conflict with schema field names."""
303+
if not field_name:
304+
raise ValueError("Undefined name")
305+
if field_name in partition_names:
306+
raise ValueError(f"Partition name has to be unique: {field_name}")
307+
303308
try:
304309
schema_field = schema.find_field(field_name)
305310
except ValueError:
@@ -311,14 +316,11 @@ def validate_partition_name(
311316
raise ValueError(f"Cannot create identity partition sourced from different field in schema: {field_name}")
312317
else:
313318
raise ValueError(f"Cannot create partition with a name that exists in schema: {field_name}")
314-
if not field_name:
315-
raise ValueError("Undefined name")
316-
if field_name in partition_names:
317-
raise ValueError(f"Partition name has to be unique: {field_name}")
318319

319320

320321
def assign_fresh_partition_spec_ids(spec: PartitionSpec, old_schema: Schema, fresh_schema: Schema) -> PartitionSpec:
321322
partition_fields = []
323+
partition_names: set[str] = set()
322324
for pos, field in enumerate(spec.fields):
323325
original_column_name = old_schema.find_column_name(field.source_id)
324326
if original_column_name is None:
@@ -327,7 +329,8 @@ def assign_fresh_partition_spec_ids(spec: PartitionSpec, old_schema: Schema, fre
327329
if fresh_field is None:
328330
raise ValueError(f"Could not find field in fresh schema: {original_column_name}")
329331

330-
validate_partition_name(field.name, field.transform, fresh_field.field_id, fresh_schema, set())
332+
validate_partition_name(field.name, field.transform, fresh_field.field_id, fresh_schema, partition_names)
333+
partition_names.add(field.name)
331334

332335
partition_fields.append(
333336
PartitionField(

‎tests/table/test_init.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1558,6 +1558,22 @@ def test_last_assigned_partition_id(table_v2: Table) -> None:
15581558
AssertLastAssignedPartitionId(last_assigned_partition_id=1).validate(base_metadata)
15591559

15601560

1561+
def test_update_spec_rejects_duplicate_partition_name(table_v2: Table) -> None:
1562+
txn = table_v2.transaction()
1563+
txn.update_spec().add_field("y", BucketTransform(16), "y_bucket").commit()
1564+
1565+
with pytest.raises(ValueError, match="Partition name has to be unique: y_bucket"):
1566+
txn.update_spec().rename_field("x", "y_bucket").commit()
1567+
1568+
with pytest.raises(ValueError, match="Partition name has to be unique: z_bucket"):
1569+
txn.update_spec().rename_field("y_bucket", "z_bucket").add_field("z", BucketTransform(16), "z_bucket").commit()
1570+
1571+
1572+
def test_update_spec_rejects_empty_partition_name(table_v2: Table) -> None:
1573+
with pytest.raises(ValueError, match="Undefined name"):
1574+
table_v2.transaction().update_spec().add_field("z", BucketTransform(16), "").commit()
1575+
1576+
15611577
@pytest.mark.parametrize("table", [lf("table_v2"), lf("table_v3")])
15621578
def test_update_spec_reuses_removed_partition_field_id(table: Table) -> None:
15631579
txn = table.transaction()

‎tests/table/test_metadata.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,27 @@ def test_new_table_metadata_format_v2_with_v3_schema_fails(field_type: Primitive
880880
)
881881

882882

883+
def test_new_table_metadata_with_duplicate_partition_name_fails() -> None:
884+
schema = Schema(
885+
NestedField(field_id=10, name="foo", field_type=StringType(), required=False),
886+
NestedField(field_id=22, name="bar", field_type=IntegerType(), required=False),
887+
)
888+
889+
partition_spec = PartitionSpec(
890+
PartitionField(source_id=10, field_id=1000, transform=IdentityTransform(), name="part"),
891+
PartitionField(source_id=22, field_id=1001, transform=IdentityTransform(), name="part"),
892+
)
893+
894+
with pytest.raises(ValueError, match="Partition name has to be unique: part"):
895+
new_table_metadata(
896+
schema=schema,
897+
partition_spec=partition_spec,
898+
sort_order=SortOrder(),
899+
location="s3://some_v1_location/",
900+
properties={"format-version": "2"},
901+
)
902+
903+
883904
def test_encrypted_key_minimal() -> None:
884905
# Mirrors Java's TestEncryptedKeyParser, where the key metadata is base64 of b"key"
885906
key = EncryptedKey.model_validate_json('{"key-id": "a", "encrypted-key-metadata": "a2V5"}')

0 commit comments

Comments
 (0)