Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 57 additions & 6 deletions pyiceberg/table/update/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,25 @@
UpdateTableMetadata,
)
from pyiceberg.typedef import L, TableVersion
from pyiceberg.types import IcebergType, ListType, MapType, NestedField, PrimitiveType, StructType
from pyiceberg.types import (
BinaryType,
DecimalType,
DoubleType,
FixedType,
FloatType,
IcebergType,
IntegerType,
ListType,
LongType,
MapType,
NestedField,
PrimitiveType,
StringType,
StructType,
UnknownType,
UUIDType,
)
from pyiceberg.utils.deprecated import deprecation_message

if TYPE_CHECKING:
import pyarrow as pa
Expand All @@ -74,6 +92,34 @@ class _Move:
other_field_id: int | None = None


def _is_valid_promotion(from_type: IcebergType, to_type: IcebergType) -> bool:

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if we can re-use promote:

def promote(file_type: IcebergType, read_type: IcebergType) -> IcebergType:

"""Check a column type change against the spec's valid primitive type promotions.

Do not use promote() here: it is the read-side rule and also accepts string <-> binary and fixed(16) -> uuid.
"""
if from_type == to_type:
return True
if isinstance(from_type, UnknownType):
# Unknown can be promoted to any primitive type in v3+
return isinstance(to_type, PrimitiveType)
if isinstance(from_type, IntegerType):
return isinstance(to_type, LongType)
if isinstance(from_type, FloatType):
return isinstance(to_type, DoubleType)
if isinstance(from_type, DecimalType) and isinstance(to_type, DecimalType):
return from_type.scale == to_type.scale and from_type.precision <= to_type.precision
return False


def _is_deprecated_promotion(from_type: IcebergType, to_type: IcebergType) -> bool:
"""Check for a non-spec type change that update_column still applies, with a deprecation warning."""
if isinstance(from_type, StringType) and isinstance(to_type, BinaryType):
return True
if isinstance(from_type, BinaryType) and isinstance(to_type, StringType):
return True
Comment on lines +116 to +119

@Fokko Fokko Nov 18, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was an earlier discussion: apache/iceberg#5151

I would just allow this for now. We should have one single place to do this, for example, promote() as suggested above.

Same for the UUID. I believe we support this because of older versions of PyArrow, that didn't support UUIDs yet. This makes it a bit more lenient to still convert it into a UUID.

Let's encode these edge cases as comments in the code, so the LLM doesn't trip over it again.

return isinstance(from_type, FixedType) and len(from_type) == 16 and isinstance(to_type, UUIDType)


class UpdateSchema(UpdateTableMetadata["UpdateSchema"]):
_schema: Schema
_last_column_id: itertools.count[int]
Expand Down Expand Up @@ -470,11 +516,16 @@ def update_column(
if not field.field_type.is_primitive:
raise ValidationError(f"Cannot change column type: {field.field_type} is not a primitive")

if not self._allow_incompatible_changes and field.field_type != field_type:
try:
promote(field.field_type, field_type)
except ResolveError as e:
raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}") from e
if not self._allow_incompatible_changes and not _is_valid_promotion(field.field_type, field_type):
if not _is_deprecated_promotion(field.field_type, field_type):
raise ValidationError(f"Cannot change column type: {full_name}: {field.field_type} -> {field_type}")
deprecation_message(
deprecated_in="0.13.0",
removed_in="0.14.0",
help_message=f"Column {full_name} changes from {field.field_type} to {field_type}, "
"which is not a valid type promotion. Pass allow_incompatible_changes=True to update_schema() "
"to keep this change.",
)

# if other updates for the same field exist in one transaction:
if updated := self._updates.get(field.field_id):
Expand Down
28 changes: 22 additions & 6 deletions tests/integration/test_rest_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
# pylint:disable=redefined-outer-name
from contextlib import nullcontext
from datetime import UTC, date, datetime, time
from decimal import Decimal
from typing import Any
Expand Down Expand Up @@ -597,15 +598,31 @@ def test_update_types_case_insensitive(catalog: Catalog) -> None:
)


allowed_promotions = [
# update_column still applies these with a deprecation warning
deprecated_promotions = [
(StringType(), BinaryType()),
(BinaryType(), StringType()),
]

allowed_promotions = [
*deprecated_promotions,
(IntegerType(), LongType()),
(FloatType(), DoubleType()),
(DecimalType(9, 2), DecimalType(18, 2)),
]


def _update_bar_type(tbl: Table, from_type: PrimitiveType, to_type: PrimitiveType) -> None:
expect_warning = (
pytest.warns(DeprecationWarning, match=r"Deprecated in 0\.13\.0, will be removed in 0\.14\.0")
if (from_type, to_type) in deprecated_promotions
else nullcontext()
)
with expect_warning:
with tbl.update_schema() as schema_update:
schema_update.update_column("bar", to_type)


@pytest.mark.parametrize("from_type, to_type", allowed_promotions, ids=str)
@pytest.mark.integration
def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catalog: Catalog) -> None:
Expand All @@ -616,8 +633,7 @@ def test_allowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, catal
),
)

with tbl.update_schema() as schema_update:
schema_update.update_column("bar", to_type)
_update_bar_type(tbl, from_type, to_type)

assert tbl.schema() == Schema(NestedField(field_id=1, name="bar", field_type=to_type, required=True))

Expand Down Expand Up @@ -663,8 +679,7 @@ def test_disallowed_updates(from_type: PrimitiveType, to_type: PrimitiveType, ca

assert str(exc_info.value).startswith("Cannot change column type: bar:")
else:
with tbl.update_schema() as schema_update:
schema_update.update_column("bar", to_type)
_update_bar_type(tbl, from_type, to_type)

assert tbl.schema() == Schema(
NestedField(field_id=1, name="bar", field_type=to_type, required=True),
Expand Down Expand Up @@ -1591,8 +1606,9 @@ def test_add_field_to_map_key(nested_table: Table) -> None:

@pytest.mark.integration
def test_alter_map_key(nested_table: Table) -> None:
# The key change is not a valid promotion, so opt in to reach the map key check
with pytest.raises(ValueError) as exc_info:
with nested_table.update_schema() as schema_update:
with nested_table.update_schema(allow_incompatible_changes=True) as schema_update:
schema_update.update_column(("quux", "key"), BinaryType())

assert "Cannot update map keys" in str(exc_info.value)
Expand Down
143 changes: 143 additions & 0 deletions tests/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,14 @@
# specific language governing permissions and limitations
# under the License.

import warnings
from textwrap import dedent
from typing import Any

import pyarrow as pa
import pytest

from pyiceberg.catalog import Catalog
from pyiceberg.exceptions import ResolveError, ValidationError
from pyiceberg.schema import (
Accessor,
Expand Down Expand Up @@ -1328,6 +1330,147 @@ def test_detect_invalid_promotion_string_to_float(table_v2: Table) -> None:
_ = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).union_by_name(new_schema)._apply()


def test_update_column_rejects_invalid_promotions(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.update_column_invalid_{catalog.name}"
tbl = catalog.create_table(
identifier,
Schema(
NestedField(field_id=1, name="name", field_type=StringType(), required=False),
NestedField(field_id=2, name="qty", field_type=IntegerType(), required=False),
NestedField(field_id=3, name="price", field_type=DecimalType(9, 2), required=False),
NestedField(field_id=4, name="key", field_type=FixedType(15), required=False),
NestedField(field_id=5, name="ratio", field_type=FloatType(), required=False),
),
)
metadata_location = tbl.metadata_location

# Not valid promotions in the spec, so the update is rejected and the table is not changed
rejected = [
("qty", FloatType(), "int -> float"),
("price", DecimalType(9, 3), "decimal(9, 2) -> decimal(9, 3)"),
("price", DecimalType(5, 2), "decimal(9, 2) -> decimal(5, 2)"),
("key", UUIDType(), "fixed[15] -> uuid"),
("ratio", IntegerType(), "float -> int"),
]
for name, new_type, promotion in rejected:
with pytest.raises(ValidationError) as exc_info:
tbl.update_schema().update_column(name, field_type=new_type).commit()
assert str(exc_info.value) == f"Cannot change column type: {name}: {promotion}"

assert catalog.load_table(identifier).metadata_location == metadata_location

# Valid promotions in the spec, and a type that does not change
for name, new_type in [("name", StringType()), ("qty", LongType()), ("price", DecimalType(18, 2)), ("ratio", DoubleType())]:
tbl.update_schema().update_column(name, field_type=new_type).commit()

assert catalog.load_table(identifier).schema() == Schema(
NestedField(field_id=1, name="name", field_type=StringType(), required=False),
NestedField(field_id=2, name="qty", field_type=LongType(), required=False),
NestedField(field_id=3, name="price", field_type=DecimalType(18, 2), required=False),
NestedField(field_id=4, name="key", field_type=FixedType(15), required=False),
NestedField(field_id=5, name="ratio", field_type=DoubleType(), required=False),
)


def test_update_column_warns_for_non_spec_promotions(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.update_column_deprecated_{catalog.name}"
tbl = catalog.create_table(
identifier,
Schema(
NestedField(field_id=1, name="name", field_type=StringType(), required=False),
NestedField(field_id=2, name="blob", field_type=BinaryType(), required=False),
NestedField(field_id=3, name="key", field_type=FixedType(16), required=False),
),
)

# Not in the spec, but still applied with a deprecation warning
for name, old_type, new_type in [
("name", StringType(), BinaryType()),
("blob", BinaryType(), StringType()),
("key", FixedType(16), UUIDType()),
]:
with pytest.warns(DeprecationWarning) as record:
tbl.update_schema().update_column(name, field_type=new_type).commit()
assert [str(w.message) for w in record if issubclass(w.category, DeprecationWarning)] == [
f"Deprecated in 0.13.0, will be removed in 0.14.0. Column {name} changes from {old_type} to {new_type}, "
"which is not a valid type promotion. Pass allow_incompatible_changes=True to update_schema() to keep this change."
]

assert catalog.load_table(identifier).schema() == Schema(
NestedField(field_id=1, name="name", field_type=BinaryType(), required=False),
NestedField(field_id=2, name="blob", field_type=StringType(), required=False),
NestedField(field_id=3, name="key", field_type=UUIDType(), required=False),
)


def test_update_column_no_warning_with_allow_incompatible_changes(catalog: Catalog) -> None:
catalog.create_namespace("default")
identifier = f"default.update_column_opt_in_{catalog.name}"
catalog.create_table(
identifier,
Schema(
NestedField(field_id=1, name="name", field_type=StringType(), required=False),
NestedField(field_id=2, name="blob", field_type=BinaryType(), required=False),
NestedField(field_id=3, name="key", field_type=FixedType(16), required=False),
),
)

# The explicit opt-in applies the same changes without a warning
for name, new_type in [("name", BinaryType()), ("blob", StringType()), ("key", UUIDType())]:
with warnings.catch_warnings(record=True) as record:
warnings.simplefilter("always")
catalog.load_table(identifier).update_schema(allow_incompatible_changes=True).update_column(
name, field_type=new_type
).commit()
assert not [w for w in record if issubclass(w.category, DeprecationWarning)]

assert catalog.load_table(identifier).schema() == Schema(
NestedField(field_id=1, name="name", field_type=BinaryType(), required=False),
NestedField(field_id=2, name="blob", field_type=StringType(), required=False),
NestedField(field_id=3, name="key", field_type=UUIDType(), required=False),
)


def test_update_column_promotes_unknown_to_primitive_only(table_v3: Table) -> None:
current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=UnknownType(), required=False))

applied = (
UpdateSchema(transaction=Transaction(table_v3), schema=current_schema)
.update_column("aCol", field_type=StringType())
._apply()
)
assert applied.find_field("aCol").field_type == StringType()

with pytest.raises(ValidationError) as exc_info:
UpdateSchema(transaction=Transaction(table_v3), schema=current_schema).update_column(
"aCol", field_type=ListType(element_id=2, element_type=StringType(), element_required=False)
)
assert str(exc_info.value) == "Cannot change column type: aCol: unknown -> list<string>"


def test_union_by_name_fixed_uuid_and_string_binary(table_v2: Table) -> None:
fixed_list = ListType(element_id=2, element_type=FixedType(16), element_required=False)
uuid_list = ListType(element_id=2, element_type=UUIDType(), element_required=False)
# An incoming UUID over an existing fixed(16) field warns, also for a list element. The other
# pairs keep the existing type without a warning.
for current_type, new_type, applied_type, warns in [
(FixedType(16), UUIDType(), UUIDType(), True),
(UUIDType(), FixedType(16), UUIDType(), False),
(StringType(), BinaryType(), StringType(), False),
(BinaryType(), StringType(), BinaryType(), False),
(fixed_list, uuid_list, uuid_list, True),
]:
current_schema = Schema(NestedField(field_id=1, name="aCol", field_type=current_type, required=False))
new_schema = Schema(NestedField(field_id=1, name="aCol", field_type=new_type, required=False))
with warnings.catch_warnings(record=True) as record:
warnings.simplefilter("always")
applied = UpdateSchema(transaction=Transaction(table_v2), schema=current_schema).union_by_name(new_schema)._apply()
assert bool([w for w in record if issubclass(w.category, DeprecationWarning)]) == warns
assert applied.find_field("aCol").field_type == applied_type


# decimal(P,S) Fixed-point decimal; precision P, scale S -> Scale is fixed [1],
# precision must be 38 or less
def test_type_promote_decimal_to_fixed_scale_with_wider_precision(table_v2: Table) -> None:
Expand Down
Loading