diff --git a/pyiceberg/table/name_mapping.py b/pyiceberg/table/name_mapping.py index bd8b8529c5..9ac197bcec 100644 --- a/pyiceberg/table/name_mapping.py +++ b/pyiceberg/table/name_mapping.py @@ -187,35 +187,20 @@ def __init__(self, updates: dict[int, NestedField], adds: dict[int, list[NestedF self._adds = adds @staticmethod - def _remove_reassigned_names(field: MappedField, assignments: dict[str, int]) -> MappedField | None: - removed_names = set() - for name in field.names: - if (assigned_id := assignments.get(name)) and assigned_id != field.field_id: - removed_names.add(name) - - remaining_names = [f for f in field.names if f not in removed_names] - if remaining_names: - return MappedField(field_id=field.field_id, names=remaining_names, fields=field.fields) - else: - return None + def _check_names_not_reassigned(fields: list[MappedField], assignments: dict[str, int], operation: str) -> None: + for field in fields: + for name in field.names: + if (assigned_id := assignments.get(name)) is not None and assigned_id != field.field_id: + raise ValueError(f"Cannot {operation} '{name}': the name mapping already assigns it to another field") def _add_new_fields(self, mapped_fields: list[MappedField], parent_id: int) -> list[MappedField]: if fields_to_add := self._adds.get(parent_id): - fields: list[MappedField] = [] - new_fields: list[MappedField] = [] - - for add in fields_to_add: - new_fields.append( - MappedField(field_id=add.field_id, names=[add.name], fields=visit(add.field_type, _CreateMapping())) - ) - - reassignments = {f.name: f.field_id for f in fields_to_add} - fields = [ - updated_field - for field in mapped_fields - if (updated_field := self._remove_reassigned_names(field, reassignments)) is not None - ] + new_fields - return fields + new_fields = [ + MappedField(field_id=add.field_id, names=[add.name], fields=visit(add.field_type, _CreateMapping())) + for add in fields_to_add + ] + self._check_names_not_reassigned(mapped_fields, {field.name: field.field_id for field in fields_to_add}, "add column") + return mapped_fields + new_fields else: return mapped_fields @@ -225,14 +210,11 @@ def mapping(self, nm: NameMapping, field_results: list[MappedField]) -> list[Map def fields(self, struct: list[MappedField], field_results: list[MappedField]) -> list[MappedField]: reassignments: dict[str, int] = { update.name: update.field_id - for f in field_results - if f.field_id is not None and (update := self._updates.get(f.field_id)) - } - return [ - updated_field for field in field_results - if (updated_field := self._remove_reassigned_names(field, reassignments)) is not None - ] + if field.field_id is not None and (update := self._updates.get(field.field_id)) + } + self._check_names_not_reassigned(field_results, reassignments, "rename column to") + return field_results def field(self, field: MappedField, field_result: list[MappedField]) -> MappedField: if field.field_id is None: diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index 1ed9ea4096..ff72c2726f 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -26,6 +26,7 @@ from typing import Any import pyarrow as pa +import pyarrow.parquet as pq import pytest from pydantic_core import ValidationError from pytest_lazy_fixtures import lf @@ -338,6 +339,27 @@ def test_write_pyarrow_schema(catalog: Catalog, test_table_identifier: Identifie table.append(pyarrow_table) +def test_add_column_refuses_name_still_used_by_name_mapping( + catalog_with_warehouse: Catalog, + tmp_path: Path, +) -> None: + imported = tmp_path / "imported.parquet" + pq.write_table(pa.table({"a": [1, 3]}), imported) + + catalog_with_warehouse.create_namespace("default") + table = catalog_with_warehouse.create_table(("default", "imported"), Schema(NestedField(1, "a", LongType()))) + table.add_files([f"file://{imported}"]) + + with table.update_schema() as update: + update.rename_column("a", "b") + + with pytest.raises(ValueError, match="name mapping already assigns it to another field"): + with table.update_schema() as update: + update.add_column("a", LongType()) + + assert table.scan().to_arrow().to_pydict() == {"b": [1, 3]} + + @pytest.mark.parametrize( "schema,expected", [ diff --git a/tests/table/test_name_mapping.py b/tests/table/test_name_mapping.py index 073e893455..fea5bbac4b 100644 --- a/tests/table/test_name_mapping.py +++ b/tests/table/test_name_mapping.py @@ -303,7 +303,7 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None: updates = {1: NestedField(1, "foo_update", StringType(), True)} adds = { -1: [NestedField(18, "add_18", StringType(), True)], - 15: [NestedField(19, "name", StringType(), True), NestedField(20, "add_20", StringType(), True)], + 15: [NestedField(19, "add_19", StringType(), True), NestedField(20, "add_20", StringType(), True)], } expected = NameMapping( @@ -345,8 +345,9 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None: field_id=15, names=["person"], fields=[ + MappedField(field_id=16, names=["name"]), MappedField(field_id=17, names=["age"]), - MappedField(field_id=19, names=["name"]), + MappedField(field_id=19, names=["add_19"]), MappedField(field_id=20, names=["add_20"]), ], ), @@ -356,6 +357,27 @@ def test_update_mapping(table_name_mapping_nested: NameMapping) -> None: assert update_mapping(table_name_mapping_nested, updates, adds) == expected +def test_update_mapping_refuses_reassigned_name_on_add(table_name_mapping_nested: NameMapping) -> None: + adds = {-1: [NestedField(18, "foo", StringType(), True)]} + + with pytest.raises(ValueError, match="name mapping already assigns it to another field"): + update_mapping(table_name_mapping_nested, {}, adds) + + +def test_update_mapping_refuses_reassigned_name_on_nested_add(table_name_mapping_nested: NameMapping) -> None: + adds = {15: [NestedField(19, "name", StringType(), True)]} + + with pytest.raises(ValueError, match="name mapping already assigns it to another field"): + update_mapping(table_name_mapping_nested, {}, adds) + + +def test_update_mapping_refuses_reassigned_name_on_rename(table_name_mapping_nested: NameMapping) -> None: + updates = {2: NestedField(2, "foo", StringType(), True)} + + with pytest.raises(ValueError, match="name mapping already assigns it to another field"): + update_mapping(table_name_mapping_nested, updates, {}) + + def test_mapping_using_by_visitor(table_schema_nested: Schema, table_name_mapping_nested: NameMapping) -> None: schema_without_ids = Schema( NestedField(field_id=0, name="foo", field_type=StringType(), required=False),