From 20044986bc0ccc42c7ab59aec4c20688790dba63 Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Fri, 20 Nov 2026 17:57:16 +0000 Subject: [PATCH] Reject columns whose field IDs are not in the table schema on write Appending, overwriting or upserting an Arrow table whose fields carry field IDs dropped the columns whose ID is not in the table schema, such as an extra column or one that was dropped and added back, while the same input without field IDs is rejected. The write paths now pass reject_unknown_field_ids to _check_pyarrow_schema_compatible, which raises a ValueError that names each such column and its field ID. add_files is unchanged. Co-authored-by: Isaac --- pyiceberg/io/pyarrow.py | 11 ++++++ pyiceberg/table/__init__.py | 28 +++++++++++++++ tests/catalog/test_catalog_behaviors.py | 48 +++++++++++++++++++++++++ tests/io/test_pyarrow.py | 35 ++++++++++++++++++ 4 files changed, 122 insertions(+) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 20333882b5..9cc3557e07 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -3141,11 +3141,15 @@ def _check_pyarrow_schema_compatible( provided_schema: pa.Schema, downcast_ns_timestamp_to_us: bool = False, format_version: TableVersion = TableProperties.DEFAULT_FORMAT_VERSION, + *, + reject_unknown_field_ids: bool = False, ) -> None: """ Check if the `requested_schema` is compatible with `provided_schema`. Two schemas are considered compatible when they are equal in terms of the Iceberg Schema type. + `reject_unknown_field_ids` also rejects the fields of `provided_schema` whose field ID is not in `requested_schema`. + Writes set it, because the writer only writes the fields of `requested_schema`. Raises: ValueError: If the schemas are not compatible. @@ -3168,6 +3172,13 @@ def _check_pyarrow_schema_compatible( f"PyArrow table contains more columns: {', '.join(sorted(additional_names))}. " "Update the schema first (hint, use union_by_name)." ) from e + if reject_unknown_field_ids and (unknown_field_ids := provided_schema.field_ids - requested_schema.field_ids): + unknown_columns = sorted( + f"{provided_schema.find_column_name(field_id)} (field ID {field_id})" for field_id in unknown_field_ids + ) + raise ValueError( + f"PyArrow table contains columns whose field IDs are not in the table schema: {', '.join(unknown_columns)}." + ) _check_schema_compatible(requested_schema, provided_schema) diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 3ea41e3f22..b6ce0b8f89 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -569,6 +569,9 @@ def append( df: An Arrow Table or a RecordBatchReader of records to append. snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the append operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -586,6 +589,7 @@ def append( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) with self._append_snapshot_producer(snapshot_properties, branch=branch) as append_files: @@ -615,6 +619,9 @@ def dynamic_partition_overwrite( df: The Arrow dataframe that will be used to overwrite the table snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the dynamic partition overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -642,6 +649,7 @@ def dynamic_partition_overwrite( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) # If dataframe does not have data, there is no need to overwrite @@ -734,6 +742,9 @@ def overwrite( snapshot_properties: Custom properties to be added to the snapshot summary case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive branch: Branch Reference to run the overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa @@ -751,6 +762,7 @@ def overwrite( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) if overwrite_filter != AlwaysFalse(): @@ -928,6 +940,9 @@ def upsert( Returns: An UpsertResult class (contains details of rows updated and inserted) + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ try: import pyarrow as pa # noqa: F401 @@ -963,6 +978,7 @@ def upsert( provided_schema=df.schema, downcast_ns_timestamp_to_us=downcast_ns_timestamp_to_us, format_version=self.table_metadata.format_version, + reject_unknown_field_ids=True, ) # get list of rows that exist so we don't have to load the entire target table @@ -1805,6 +1821,9 @@ def upsert( Returns: An UpsertResult class (contains details of rows updated and inserted) + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: return tx.upsert( @@ -1834,6 +1853,9 @@ def append( df: An Arrow Table or a RecordBatchReader of records to append. snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the append operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.append(df=df, snapshot_properties=snapshot_properties, branch=branch) @@ -1848,6 +1870,9 @@ def dynamic_partition_overwrite( df: The Arrow dataframe that will be used to overwrite the table snapshot_properties: Custom properties to be added to the snapshot summary branch: Branch Reference to run the dynamic partition overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.dynamic_partition_overwrite(df=df, snapshot_properties=snapshot_properties, branch=branch) @@ -1880,6 +1905,9 @@ def overwrite( snapshot_properties: Custom properties to be added to the snapshot summary case_sensitive: A bool determine if the provided `overwrite_filter` is case-sensitive branch: Branch Reference to run the overwrite operation + + Raises: + ValueError: If the schema of ``df`` is not compatible with the table schema. """ with self.transaction() as tx: tx.overwrite( diff --git a/tests/catalog/test_catalog_behaviors.py b/tests/catalog/test_catalog_behaviors.py index a5099e4e6b..a34ae7efcd 100644 --- a/tests/catalog/test_catalog_behaviors.py +++ b/tests/catalog/test_catalog_behaviors.py @@ -1005,6 +1005,54 @@ def test_write_and_evolve(catalog: Catalog, format_version: int) -> None: txn.delete("foo = 'a'") +def test_write_rejects_field_ids_not_in_table_schema(catalog: Catalog) -> None: + catalog.create_namespace("default") + identifier = f"default.write_unknown_field_ids_{catalog.name}" + old_schema = Schema(NestedField(1, "id", LongType()), NestedField(2, "x", LongType()), NestedField(3, "y", LongType())) + spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="id")) + tbl = catalog.create_table(identifier, schema=old_schema, partition_spec=spec) + # Drop x and add it back, so the new x has field ID 4, and rename y (field ID 3) to w + with tbl.update_schema() as update: + update.delete_column("x") + update.rename_column("y", "w") + with tbl.update_schema() as update: + update.add_column("x", LongType()) + + writes: list[Callable[[pa.Table], Any]] = [ + tbl.append, + tbl.overwrite, + tbl.dynamic_partition_overwrite, + # The table is empty and inserts are off, so only the check in upsert itself can reject the input + lambda df: tbl.upsert(df, join_cols=["id"], when_not_matched_insert_all=False), + ] + schema_with_z = Schema(*tbl.schema().fields, NestedField(99, "z", LongType())) + for input_schema, unknown_columns in [(schema_with_z, "z (field ID 99)"), (old_schema, "x (field ID 2)")]: + df = pa.Table.from_pylist([{"id": 1, "x": 2, "y": 3, "w": 3, "z": 4}], schema=input_schema.as_arrow()) + for write in writes: + with pytest.raises(ValueError) as exc_info: + write(df) + assert ( + str(exc_info.value) + == f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}." + ) + assert catalog.load_table(identifier).metadata.snapshots == [] + + # Still accepted: y is the old name of w (field ID 3), the columns are reordered, y is an int and x is missing + tbl.append( + pa.Table.from_pylist( + [{"y": 5, "id": 1}], schema=Schema(NestedField(3, "y", IntegerType()), NestedField(1, "id", LongType())).as_arrow() + ) + ) + # Without field IDs, the columns are matched by name + tbl.append( + pa.Table.from_pylist([{"id": 2, "x": 6}], schema=pa.schema([pa.field("id", pa.int64()), pa.field("x", pa.int64())])) + ) + assert sorted(catalog.load_table(identifier).scan().to_arrow().to_pylist(), key=lambda row: row["id"]) == [ + {"id": 1, "w": 5, "x": None}, + {"id": 2, "w": None, "x": 6}, + ] + + # Merge manifests @pytest.mark.parametrize("format_version", [1, 2]) def test_merge_manifests_local_file_system(catalog: Catalog, arrow_table_with_null: pa.Table, format_version: int) -> None: diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 439405d121..7c8954b593 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -3320,6 +3320,41 @@ def test_schema_compatible_nested(table_schema_nested: Schema) -> None: pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`") +def test_schema_mismatch_unknown_field_ids() -> None: + table_schema = Schema( + NestedField(1, "id", LongType()), + NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()))), + NestedField(4, "tags", ListType(5, LongType(), element_required=False)), + NestedField(6, "props", MapType(7, StringType(), 8, LongType(), value_required=False)), + ) + for provided_schema, unknown_columns in [ + (Schema(NestedField(1, "id", LongType()), NestedField(99, "z", LongType())), "z (field ID 99)"), + # A struct field, list element and map value whose parent is in the table + ( + Schema( + NestedField(2, "payload", StructType(NestedField(3, "qty", LongType()), NestedField(99, "extra", LongType()))) + ), + "payload.extra (field ID 99)", + ), + (Schema(NestedField(4, "tags", ListType(99, LongType(), element_required=False))), "tags.element (field ID 99)"), + ( + Schema(NestedField(6, "props", MapType(7, StringType(), 99, LongType(), value_required=False))), + "props.value (field ID 99)", + ), + # A struct that is not in the table, and its field + ( + Schema(NestedField(98, "extra", StructType(NestedField(99, "qty", LongType())))), + "extra (field ID 98), extra.qty (field ID 99)", + ), + ]: + with pytest.raises(ValueError) as exc_info: + _check_pyarrow_schema_compatible(table_schema, schema_to_pyarrow(provided_schema), reject_unknown_field_ids=True) + assert ( + str(exc_info.value) + == f"PyArrow table contains columns whose field IDs are not in the table schema: {unknown_columns}." + ) + + def test_schema_mismatch_additional_field(table_schema_simple: Schema) -> None: other_schema = pa.schema( (