Skip to content

Commit c8f9ed8

Browse files
committed
Check partition names against the current schema when adding a spec
UpdateSpec checks new partition names against the schema it was loaded with. If another writer added a column with the same name in between, the commit still went through and wrote metadata Java refuses to load. Check the names again when the spec is applied to the current metadata. Closes #4198
1 parent 06a529d commit c8f9ed8

2 files changed

Lines changed: 26 additions & 2 deletions

File tree

‎pyiceberg/table/update/__init__.py‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
from pydantic import Field, field_validator, model_serializer, model_validator
2828

2929
from pyiceberg.exceptions import CommitFailedException
30-
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec
30+
from pyiceberg.partitioning import PARTITION_FIELD_ID_START, PartitionSpec, validate_partition_name
3131
from pyiceberg.schema import Schema
3232
from pyiceberg.table.metadata import (
3333
SUPPORTED_TABLE_FORMAT_VERSION,
@@ -408,6 +408,11 @@ def _(update: AddPartitionSpecUpdate, base_metadata: TableMetadata, context: _Ta
408408
if spec.spec_id == update.spec.spec_id and spec != update.spec:
409409
raise ValueError(f"Partition spec with id {spec.spec_id} already exists: {spec}")
410410

411+
# The spec was built against the schema it was loaded with, which another writer may have changed since
412+
schema = base_metadata.schema()
413+
for field in update.spec.fields:
414+
validate_partition_name(field.name, field.transform, field.source_id, schema, set())
415+
411416
metadata_updates: dict[str, Any] = {
412417
"partition_specs": base_metadata.partition_specs + [update.spec],
413418
"last_partition_id": max(

‎tests/catalog/test_catalog_behaviors.py‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
from pyiceberg.table.snapshots import Operation
4747
from pyiceberg.table.sorting import NullOrder, SortDirection, SortField, SortOrder
4848
from pyiceberg.table.update import AddSchemaUpdate, SetCurrentSchemaUpdate
49-
from pyiceberg.transforms import IdentityTransform
49+
from pyiceberg.transforms import BucketTransform, IdentityTransform
5050
from pyiceberg.typedef import Identifier
5151
from pyiceberg.types import BooleanType, DoubleType, IntegerType, LongType, NestedField, StringType
5252

@@ -622,6 +622,25 @@ def test_concurrent_commit_table(catalog: Catalog, table_schema_nested: Schema,
622622
update.add_column(path="c", field_type=IntegerType())
623623

624624

625+
def test_concurrent_update_spec_checks_names_against_current_schema(catalog: Catalog, test_table_identifier: Identifier) -> None:
626+
namespace = Catalog.namespace_from(test_table_identifier)
627+
catalog.create_namespace(namespace)
628+
table_a = catalog.create_table(test_table_identifier, Schema(NestedField(1, "x", LongType(), required=False)))
629+
table_b = catalog.load_table(test_table_identifier)
630+
631+
with table_a.update_schema() as update:
632+
update.add_column(path="xb", field_type=LongType())
633+
634+
# table_b still has the schema without xb, so only the commit can see the conflict
635+
with pytest.raises(ValueError, match="Cannot create partition with a name that exists in schema: xb"):
636+
with table_b.update_spec() as update:
637+
update.add_field("x", BucketTransform(4), "xb")
638+
639+
table = catalog.load_table(test_table_identifier)
640+
assert table.schema().column_names == ["x", "xb"]
641+
assert table.spec().is_unpartitioned()
642+
643+
625644
def test_delete_metadata_multiple(catalog: Catalog, table_schema_nested: Schema, random_table_identifier: Identifier) -> None:
626645
namespace = Catalog.namespace_from(random_table_identifier)
627646
catalog.create_namespace(namespace)

0 commit comments

Comments
 (0)