From 7ae198a9d73b07fa59fb470e9fdd85bdfbceb9d4 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Mon, 26 Oct 2026 17:08:02 -0500 Subject: [PATCH 1/5] API, Core: Reserve the file type's derived field ID block A file column owns the field IDs immediately following its own, because its nested fields are derived from the enclosing ID rather than assigned. Reject a file column whose type does not derive from its field ID, and reserve the block when a file column is added or reassigned so later additions cannot collide. Nested fields of a file column cannot be renamed, deleted, promoted, moved, or otherwise updated, since the type fixes them. Generated-by: Cursor (Claude Opus 5.5) --- .../main/java/org/apache/iceberg/Schema.java | 51 ++++++-- .../iceberg/types/CheckCompatibility.java | 10 ++ .../apache/iceberg/types/TestFileType.java | 46 +++++++ .../iceberg/types/TestReadabilityChecks.java | 21 ++++ .../org/apache/iceberg/MetricsConfig.java | 5 + .../java/org/apache/iceberg/SchemaOrder.java | 5 + .../java/org/apache/iceberg/SchemaUpdate.java | 33 ++++- .../iceberg/TestFileTypeTableMetadata.java | 105 ++++++++++++++++ .../org/apache/iceberg/TestSchemaUpdate.java | 115 ++++++++++++++++++ 9 files changed, 378 insertions(+), 13 deletions(-) create mode 100644 core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index eab56dd7ada3..946ab13569e6 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -149,6 +149,8 @@ public Schema( this.struct = StructType.of(finalColumns); this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null; + validateFileBaseIds(); + // validate IdentifierField if (identifierFieldIds != null) { Map idToParent = TypeUtil.indexParents(struct); @@ -161,6 +163,20 @@ public Schema( this.highestFieldId = lazyIdToName().keySet().stream().mapToInt(i -> i).max().orElse(0); } + private void validateFileBaseIds() { + for (NestedField field : lazyIdToField().values()) { + if (field.type().isFileType()) { + int baseId = field.type().asFileType().baseId(); + Preconditions.checkArgument( + baseId == field.fieldId(), + "Invalid file column %s: file base ID %s does not match field ID %s", + lazyIdToName().get(field.fieldId()), + baseId, + field.fieldId()); + } + } + } + static void validateIdentifierField( int fieldId, Map idToField, Map idToParent) { Types.NestedField field = idToField.get(fieldId); @@ -603,20 +619,33 @@ private List reassignIds(List columns, TypeUtil.GetID if (getID == null) { return columns; } - Type res = - TypeUtil.assignIds( - StructType.of(columns), - oldId -> { - int newId = getID.get(oldId); - if (newId != oldId) { - idsToReassigned.put(oldId, newId); - idsToOriginal.put(newId, oldId); - } - return newId; - }); + + TypeUtil.GetID tracked = + new TypeUtil.GetID() { + @Override + public int get(int oldId) { + return track(oldId, getID.get(oldId)); + } + + @Override + public int get(int oldId, Type type) { + return track(oldId, getID.get(oldId, type)); + } + }; + + Type res = TypeUtil.assignIds(StructType.of(columns), tracked); return res.asStructType().fields(); } + private int track(int oldId, int newId) { + if (newId != oldId) { + idsToReassigned.put(oldId, newId); + idsToOriginal.put(newId, oldId); + } + + return newId; + } + /** * Check the compatibility of the schema with a format version. * diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 3b3a38ff5aeb..333fbced8dcd 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -260,6 +260,16 @@ public List variant(Types.VariantType readVariant) { return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readVariant)); } + @Override + public List file(Types.FileType readFile, Iterable> fieldErrorLists) { + if (currentType.isFileType()) { + // the nested fields are derived from the base id, so matching ids means matching fields + return NO_ERRORS; + } + + return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readFile)); + } + @Override public List primitive(Type.PrimitiveType readPrimitive) { if (currentType.equals(readPrimitive)) { diff --git a/api/src/test/java/org/apache/iceberg/types/TestFileType.java b/api/src/test/java/org/apache/iceberg/types/TestFileType.java index 661ac5629070..39727ee26299 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestFileType.java +++ b/api/src/test/java/org/apache/iceberg/types/TestFileType.java @@ -64,6 +64,52 @@ void findsNestedFieldsByName() { assertThat(FILE.fieldType("missing")).isNull(); } + @Test + void rejectsASchemaWhereAnotherColumnHoldsADerivedId() { + assertThatThrownBy( + () -> + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(3, "data", Types.StringType.get()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContainingAll("3=photo.uri", "3=data"); + } + + @Test + void rejectsASchemaWhereTwoFileColumnsOverlap() { + assertThatThrownBy( + () -> + new Schema( + optional(1, "photo", Types.FileType.of(1)), + optional(2, "thumbnail", Types.FileType.of(2)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContaining("thumbnail"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIds() { + assertThatThrownBy(() -> new Schema(optional(5, "photo", Types.FileType.of(9)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column photo: file base ID 9 does not match field ID 5"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIdsInAMap() { + assertThatThrownBy( + () -> + new Schema( + optional( + 1, + "byName", + Types.MapType.ofOptional( + 2, 3, Types.StringType.get(), Types.FileType.of(9))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column byName.value: file base ID 9 does not match field ID 3"); + } + @Test void isDistinguishableFromAStruct() { assertThat(FILE.isFileType()).isTrue(); diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 20299cdafce2..1aad90ac240a 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -286,6 +286,27 @@ public void testIncompatibleStructAndPrimitive() { .contains("struct cannot be read as a string"); } + @Test + void incompatibleFileAndStruct() { + Schema fileSchema = new Schema(optional(1, "photo", Types.FileType.of(1))); + Schema structSchema = + new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields()))); + + List asFile = CheckCompatibility.readCompatibilityErrors(fileSchema, structSchema); + assertThat(asFile).hasSize(1); + assertThat(asFile.get(0)) + .as("Should complain that a struct cannot be read as a file") + .contains("cannot be read as a file"); + + List asStruct = CheckCompatibility.readCompatibilityErrors(structSchema, fileSchema); + assertThat(asStruct).hasSize(1); + assertThat(asStruct.get(0)) + .as("Should complain that a file cannot be read as a struct") + .contains("file cannot be read as a struct"); + + assertThat(CheckCompatibility.readCompatibilityErrors(fileSchema, fileSchema)).isEmpty(); + } + @Test public void testMultipleErrors() { // required field is optional and cannot be promoted to the read type diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index f438c1bc5648..0a5664461e8b 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -190,6 +190,11 @@ public Set struct(Types.StructType struct, Iterable> field return null; } + @Override + public Set file(Types.FileType file, Iterable> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override @SuppressWarnings("ReturnValueIgnored") public Set field(Types.NestedField field, Supplier> fieldResult) { diff --git a/core/src/main/java/org/apache/iceberg/SchemaOrder.java b/core/src/main/java/org/apache/iceberg/SchemaOrder.java index 4f5559fbae3e..730ade9721f6 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaOrder.java +++ b/core/src/main/java/org/apache/iceberg/SchemaOrder.java @@ -110,6 +110,11 @@ public List map( return null; } + @Override + public List file(Types.FileType file, Iterable> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public List variant(Types.VariantType variant) { return null; diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 0aed742f705f..c73979c7abc4 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -63,6 +63,7 @@ class SchemaUpdate implements UpdateSchema { private final Map addedNameToId = Maps.newHashMap(); private final Multimap moves = Multimaps.newListMultimap(Maps.newHashMap(), Lists::newArrayList); + private final TypeUtil.NextID nextColumnId = this::assignNewColumnId; private int lastColumnId; private boolean allowIncompatibleChanges = false; private Set identifierFieldNames; @@ -163,7 +164,7 @@ private void internalAddColumn( fullName); // assign new IDs in order - int newId = assignNewColumnId(); + int newId = nextColumnId.get(type); // update tracking for moves addedNameToId.put(caseSensitivityAwareName(fullName), newId); @@ -176,7 +177,7 @@ private void internalAddColumn( .withName(name) .isOptional(isOptional) .withId(newId) - .ofType(TypeUtil.assignFreshIds(type, this::assignNewColumnId)) + .ofType(typeForNewColumn(type, newId)) .withDoc(doc) .withInitialDefault(defaultValue) .withWriteDefault(defaultValue) @@ -186,10 +187,19 @@ private void internalAddColumn( parentToAddedIds.put(parentId, newId); } + private Type typeForNewColumn(Type type, int fieldId) { + if (type.isFileType()) { + return Types.FileType.of(fieldId); + } + + return TypeUtil.assignFreshIds(type, nextColumnId); + } + @Override public UpdateSchema deleteColumn(String name) { Types.NestedField field = findField(name); Preconditions.checkArgument(field != null, "Cannot delete missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !parentToAddedIds.containsKey(field.fieldId()), "Cannot delete a column that has additions: %s", @@ -205,6 +215,7 @@ public UpdateSchema deleteColumn(String name) { public UpdateSchema renameColumn(String name, String newName) { Types.NestedField field = findField(name); Preconditions.checkArgument(field != null, "Cannot rename missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument(newName != null, "Cannot rename a column to null"); Preconditions.checkArgument( !deletes.contains(field.fieldId()), @@ -241,6 +252,7 @@ public UpdateSchema makeColumnOptional(String name) { private void internalUpdateColumnRequirement(String name, boolean isOptional) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); if ((!isOptional && field.isRequired()) || (isOptional && field.isOptional())) { // if the change is a noop, allow it even if allowIncompatibleChanges is false @@ -273,6 +285,7 @@ private void internalUpdateColumnRequirement(String name, boolean isOptional) { public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -301,6 +314,7 @@ public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { public UpdateSchema updateColumnDoc(String name, String doc) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -322,6 +336,7 @@ public UpdateSchema updateColumnDoc(String name, String doc) { public UpdateSchema updateColumnDefault(String name, Literal newDefault) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -396,6 +411,15 @@ private boolean isAdded(String name) { return addedNameToId.containsKey(caseSensitivityAwareName(name)); } + private void checkNotNestedInFile(String name, int fieldId) { + Integer parentId = idToParent.get(fieldId); + Types.NestedField parent = parentId != null ? schema.findField(parentId) : null; + Preconditions.checkArgument( + parent == null || !parent.type().isFileType(), + "Cannot change a nested field of a file column: %s", + name); + } + private Types.NestedField findForUpdate(String name) { Types.NestedField existing = findField(name); if (existing != null) { @@ -655,6 +679,11 @@ public Type struct(Types.StructType struct, List fieldResults) { return struct; } + @Override + public Type file(Types.FileType file, List fieldResults) { + return file; + } + @Override public Type field(Types.NestedField field, Type fieldResult) { // the API validates deletes, updates, and additions don't conflict diff --git a/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java new file mode 100644 index 000000000000..75ef59abe62a --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class TestFileTypeTableMetadata { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + + @ParameterizedTest + @ValueSource(ints = {1, 2, 3}) + void rejectsAFileColumnBeforeFormatVersion4(int formatVersion) { + assertThatThrownBy(() -> newTableMetadata(formatVersion)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Invalid type for photo: file is not supported until v4"); + } + + @Test + void acceptsAMetricsModeForAFileNestedField() { + TableMetadata metadata = + TableMetadata.newTableMetadata( + SCHEMA, + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of( + TableProperties.FORMAT_VERSION, + "4", + TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", + "full")); + + assertThat(metadata.properties()) + .containsEntry(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", "full"); + } + + @Test + void replacementReservesDerivedIdsWhenAColumnBecomesAFile() { + TableMetadata metadata = + TableMetadata.newTableMetadata( + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.StringType.get()), + optional(3, "data", Types.StringType.get())), + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); + Schema updated = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101)), + optional(108, "data", Types.StringType.get()), + optional(109, "notes", Types.StringType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(4, "photo", Types.FileType.of(4)), + optional(3, "data", Types.StringType.get()), + optional(11, "notes", Types.StringType.get())); + + TableMetadata replacement = + metadata.buildReplacement( + updated, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + metadata.location(), + ImmutableMap.of()); + + assertThat(replacement.schema().asStruct()).isEqualTo(expected.asStruct()); + assertThat(replacement.lastColumnId()).isEqualTo(expected.highestFieldId()); + } + + private static TableMetadata newTableMetadata(int formatVersion) { + return TableMetadata.newTableMetadata( + SCHEMA, + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index 5325e4013c68..d95a4925449b 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -2590,4 +2590,119 @@ public void testCaseInsensitiveMoveAfterNewlyAddedField() { assertThat(actual.asStruct()).isEqualTo(expected.asStruct()); } + + private static final Schema FILE_SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + private static SchemaUpdate fileUpdate() { + return new SchemaUpdate(FILE_SCHEMA, FILE_SCHEMA.highestFieldId()); + } + + @Test + void cannotAddColumnToFileColumn() { + assertThatThrownBy(() -> fileUpdate().addColumn("photo", "extra", Types.StringType.get())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot add to non-struct column: photo: file"); + } + + @Test + void cannotDeleteFileNestedField() { + assertThatThrownBy(() -> fileUpdate().deleteColumn("photo.checksum")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.checksum"); + } + + @Test + void cannotRenameFileNestedField() { + assertThatThrownBy(() -> fileUpdate().renameColumn("photo.uri", "location")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotPromoteFileNestedField() { + assertThatThrownBy(() -> fileUpdate().updateColumn("photo.size", Types.LongType.get())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.size"); + } + + @Test + void cannotUpdateFileNestedFieldDoc() { + assertThatThrownBy(() -> fileUpdate().updateColumnDoc("photo.uri", "the location")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotUpdateFileNestedFieldDefault() { + assertThatThrownBy( + () -> fileUpdate().updateColumnDefault("photo.uri", Literal.of("s3://bucket/key"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotUpdateFileNestedFieldRequirement() { + assertThatThrownBy(() -> fileUpdate().requireColumn("photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + assertThatThrownBy(() -> fileUpdate().makeColumnOptional("photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotMoveFileNestedField() { + assertThatThrownBy(() -> fileUpdate().moveFirst("photo.checksum")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + assertThatThrownBy(() -> fileUpdate().moveBefore("photo.checksum", "photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + assertThatThrownBy(() -> fileUpdate().moveAfter("photo.uri", "photo.inline")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void renameFileColumn() { + Schema renamed = fileUpdate().renameColumn("photo", "image").apply(); + + assertThat(renamed.findField("image").type()).isEqualTo(Types.FileType.of(2)); + assertThat(renamed.findField("image.uri").fieldId()).isEqualTo(3); + } + + @Test + void deleteFileColumn() { + Schema deleted = fileUpdate().deleteColumn("photo").apply(); + + assertThat(deleted.findField("photo")).isNull(); + assertThat(deleted.asStruct()) + .isEqualTo( + new Schema( + required(1, "id", Types.LongType.get()), + optional(9, "data", Types.StringType.get())) + .asStruct()); + } + + @Test + void addFileColumnReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn("photo", Types.FileType.of(0)) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } } From 673bef622521cac7901575aae42723c4414971a5 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Mon, 16 Nov 2026 14:30:15 -0600 Subject: [PATCH 2/5] API, Core: Address review feedback on the file type ID block Default inferred metrics for a file column to the uri, size, and content_type fields so a file uses three slots of the inferred-column limit instead of six. The remaining nested fields default to counts only. Validate file base IDs with a schema walk instead of forcing the ID index, and tighten and split the tests that review flagged. Generated-by: Cursor --- .../main/java/org/apache/iceberg/Schema.java | 15 ++- .../iceberg/types/CheckCompatibility.java | 1 - .../java/org/apache/iceberg/types/Types.java | 34 ++++- .../java/org/apache/iceberg/TestSchema.java | 47 +++++++ .../apache/iceberg/types/TestFileType.java | 46 ------- .../iceberg/types/TestReadabilityChecks.java | 36 +++--- .../org/apache/iceberg/MetricsConfig.java | 27 +++- .../iceberg/TestFileTypeTableMetadata.java | 105 --------------- .../org/apache/iceberg/TestMetricsConfig.java | 120 ++++++++++++++++++ .../org/apache/iceberg/TestSchemaOrder.java | 21 +++ .../org/apache/iceberg/TestSchemaUpdate.java | 114 ++++++++++++++--- .../org/apache/iceberg/TestTableMetadata.java | 55 ++++++++ 12 files changed, 427 insertions(+), 194 deletions(-) delete mode 100644 core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index 946ab13569e6..4b3909a3729e 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -149,7 +149,7 @@ public Schema( this.struct = StructType.of(finalColumns); this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null; - validateFileBaseIds(); + validateFileBaseIds(struct, null); // validate IdentifierField if (identifierFieldIds != null) { @@ -163,16 +163,23 @@ public Schema( this.highestFieldId = lazyIdToName().keySet().stream().mapToInt(i -> i).max().orElse(0); } - private void validateFileBaseIds() { - for (NestedField field : lazyIdToField().values()) { + private void validateFileBaseIds(Type type, String parentName) { + if (!type.isNestedType()) { + return; + } + + for (NestedField field : type.asNestedType().fields()) { + String name = parentName != null ? parentName + "." + field.name() : field.name(); if (field.type().isFileType()) { int baseId = field.type().asFileType().baseId(); Preconditions.checkArgument( baseId == field.fieldId(), "Invalid file column %s: file base ID %s does not match field ID %s", - lazyIdToName().get(field.fieldId()), + name, baseId, field.fieldId()); + } else { + validateFileBaseIds(field.type(), name); } } } diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 333fbced8dcd..dc6f11c07e21 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -263,7 +263,6 @@ public List variant(Types.VariantType readVariant) { @Override public List file(Types.FileType readFile, Iterable> fieldErrorLists) { if (currentType.isFileType()) { - // the nested fields are derived from the base id, so matching ids means matching fields return NO_ERRORS; } diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index 3334fe38c41f..62df4c235c44 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -1159,6 +1159,16 @@ public static final class FileType extends NestedType { public static final String NAME = "file"; public static final int NUM_NESTED_FIELDS = 6; + private static final int URI_ID_OFFSET = 1; + private static final int OFFSET_ID_OFFSET = 2; + private static final int SIZE_ID_OFFSET = 3; + private static final int CONTENT_TYPE_ID_OFFSET = 4; + private static final int CHECKSUM_ID_OFFSET = 5; + private static final int INLINE_ID_OFFSET = 6; + private static final int[] DEFAULT_INFERRED_METRIC_OFFSETS = { + URI_ID_OFFSET, SIZE_ID_OFFSET, CONTENT_TYPE_ID_OFFSET + }; + private static final String URI = "uri"; private static final String OFFSET = "offset"; private static final String SIZE = "size"; @@ -1181,6 +1191,17 @@ public static FileType of(int baseId) { return new FileType(baseId); } + /** + * Returns the offsets of nested fields that receive inferred metrics by default. + * + *

Offsets are relative to {@link #baseId()}. + * + * @return nested field offsets with inferred metrics + */ + public static int[] defaultInferredMetricOffsets() { + return DEFAULT_INFERRED_METRIC_OFFSETS; + } + private final int baseId; // Lazy and derived from baseId @@ -1284,12 +1305,13 @@ private List lazyFieldList() { if (fieldList == null) { this.fieldList = ImmutableList.of( - NestedField.optional(baseId + 1, URI, StringType.get()), - NestedField.optional(baseId + 2, OFFSET, LongType.get()), - NestedField.optional(baseId + 3, SIZE, LongType.get()), - NestedField.optional(baseId + 4, CONTENT_TYPE, StringType.get()), - NestedField.optional(baseId + 5, CHECKSUM, StringType.get()), - NestedField.optional(baseId + 6, INLINE, BinaryType.get())); + NestedField.optional(baseId + URI_ID_OFFSET, URI, StringType.get()), + NestedField.optional(baseId + OFFSET_ID_OFFSET, OFFSET, LongType.get()), + NestedField.optional(baseId + SIZE_ID_OFFSET, SIZE, LongType.get()), + NestedField.optional( + baseId + CONTENT_TYPE_ID_OFFSET, CONTENT_TYPE, StringType.get()), + NestedField.optional(baseId + CHECKSUM_ID_OFFSET, CHECKSUM, StringType.get()), + NestedField.optional(baseId + INLINE_ID_OFFSET, INLINE, BinaryType.get())); } return fieldList; diff --git a/api/src/test/java/org/apache/iceberg/TestSchema.java b/api/src/test/java/org/apache/iceberg/TestSchema.java index e410ee682991..16fc5b97af85 100644 --- a/api/src/test/java/org/apache/iceberg/TestSchema.java +++ b/api/src/test/java/org/apache/iceberg/TestSchema.java @@ -338,4 +338,51 @@ public void testIndexFieldsNestedSchema() { assertThat(fields.get(5).name()).isEqualTo("email"); assertThat(((Types.StructType) fields.get(2).type()).fields()).hasSize(3); } + + @Test + void rejectsASchemaWhereAnotherColumnHoldsADerivedId() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.FileType.of(2)), + Types.NestedField.optional(3, "data", Types.StringType.get()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContainingAll("3=photo.uri", "3=data"); + } + + @Test + void rejectsASchemaWhereTwoFileColumnsOverlap() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.optional(1, "photo", Types.FileType.of(1)), + Types.NestedField.optional(2, "thumbnail", Types.FileType.of(2)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContaining("thumbnail"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIds() { + assertThatThrownBy( + () -> new Schema(Types.NestedField.optional(5, "photo", Types.FileType.of(9)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column photo: file base ID 9 does not match field ID 5"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIdsInAMap() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.optional( + 1, + "byName", + Types.MapType.ofOptional( + 2, 3, Types.StringType.get(), Types.FileType.of(9))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column byName.value: file base ID 9 does not match field ID 3"); + } } diff --git a/api/src/test/java/org/apache/iceberg/types/TestFileType.java b/api/src/test/java/org/apache/iceberg/types/TestFileType.java index 39727ee26299..661ac5629070 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestFileType.java +++ b/api/src/test/java/org/apache/iceberg/types/TestFileType.java @@ -64,52 +64,6 @@ void findsNestedFieldsByName() { assertThat(FILE.fieldType("missing")).isNull(); } - @Test - void rejectsASchemaWhereAnotherColumnHoldsADerivedId() { - assertThatThrownBy( - () -> - new Schema( - required(1, "id", Types.LongType.get()), - optional(2, "photo", Types.FileType.of(2)), - optional(3, "data", Types.StringType.get()))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Multiple entries with same key") - .hasMessageContainingAll("3=photo.uri", "3=data"); - } - - @Test - void rejectsASchemaWhereTwoFileColumnsOverlap() { - assertThatThrownBy( - () -> - new Schema( - optional(1, "photo", Types.FileType.of(1)), - optional(2, "thumbnail", Types.FileType.of(2)))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageStartingWith("Multiple entries with same key") - .hasMessageContaining("thumbnail"); - } - - @Test - void rejectsASchemaWithUnderivedNestedIds() { - assertThatThrownBy(() -> new Schema(optional(5, "photo", Types.FileType.of(9)))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid file column photo: file base ID 9 does not match field ID 5"); - } - - @Test - void rejectsASchemaWithUnderivedNestedIdsInAMap() { - assertThatThrownBy( - () -> - new Schema( - optional( - 1, - "byName", - Types.MapType.ofOptional( - 2, 3, Types.StringType.get(), Types.FileType.of(9))))) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid file column byName.value: file base ID 9 does not match field ID 3"); - } - @Test void isDistinguishableFromAStruct() { assertThat(FILE.isFileType()).isTrue(); diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 1aad90ac240a..e057680987d1 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -61,6 +61,10 @@ public class TestReadabilityChecks { Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY), }; + private static final Schema FILE_SCHEMA = new Schema(optional(1, "photo", Types.FileType.of(1))); + private static final Schema FILE_AS_STRUCT_SCHEMA = + new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields()))); + @Test public void testPrimitiveTypes() { for (Type.PrimitiveType from : PRIMITIVES) { @@ -287,24 +291,24 @@ public void testIncompatibleStructAndPrimitive() { } @Test - void incompatibleFileAndStruct() { - Schema fileSchema = new Schema(optional(1, "photo", Types.FileType.of(1))); - Schema structSchema = - new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields()))); - - List asFile = CheckCompatibility.readCompatibilityErrors(fileSchema, structSchema); - assertThat(asFile).hasSize(1); - assertThat(asFile.get(0)) - .as("Should complain that a struct cannot be read as a file") - .contains("cannot be read as a file"); + void structCannotBeReadAsAFile() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_AS_STRUCT_SCHEMA)) + .singleElement() + .asString() + .endsWith("cannot be read as a file"); + } - List asStruct = CheckCompatibility.readCompatibilityErrors(structSchema, fileSchema); - assertThat(asStruct).hasSize(1); - assertThat(asStruct.get(0)) - .as("Should complain that a file cannot be read as a struct") - .contains("file cannot be read as a struct"); + @Test + void fileCannotBeReadAsAStruct() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_AS_STRUCT_SCHEMA, FILE_SCHEMA)) + .singleElement() + .asString() + .endsWith("file cannot be read as a struct"); + } - assertThat(CheckCompatibility.readCompatibilityErrors(fileSchema, fileSchema)).isEmpty(); + @Test + void fileCanBeReadAsAFile() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_SCHEMA)).isEmpty(); } @Test diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index 0a5664461e8b..8625385cc9d9 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -192,7 +192,13 @@ public Set struct(Types.StructType struct, Iterable> field @Override public Set file(Types.FileType file, Iterable> fieldResults) { - return struct(file.asStruct(), fieldResults); + for (int offset : Types.FileType.defaultInferredMetricOffsets()) { + if (shouldContinue()) { + idSet.add(file.baseId() + offset); + } + } + + return null; } @Override @@ -280,6 +286,8 @@ private static MetricsConfig from( Map columnModes = Maps.newHashMap(); + columnModes.putAll(fileColumnModes(schema, defaultMode)); + columnModes.putAll(defaultColumnConf); // Default sort columns to at least truncate (overridden by config) @@ -318,6 +326,23 @@ private static Map defaultColumnModes( return builder.build(); } + private static Map fileColumnModes(Schema schema, MetricsMode defaultMode) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + if (schema != null) { + MetricsMode fileMode = + defaultMode == MetricsModes.None.get() ? defaultMode : MetricsModes.Counts.get(); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.type().isFileType()) { + for (Types.NestedField nested : field.type().asFileType().fields()) { + builder.put(schema.findColumnName(nested.fieldId()), fileMode); + } + } + } + } + + return builder.build(); + } + private static Map sortColumnModes( SortOrder order, MetricsMode configuredDefault) { ImmutableMap.Builder builder = ImmutableMap.builder(); diff --git a/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java deleted file mode 100644 index 75ef59abe62a..000000000000 --- a/core/src/test/java/org/apache/iceberg/TestFileTypeTableMetadata.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.apache.iceberg.types.Types.NestedField.optional; -import static org.apache.iceberg.types.Types.NestedField.required; -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - -class TestFileTypeTableMetadata { - private static final Schema SCHEMA = - new Schema( - required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); - - @ParameterizedTest - @ValueSource(ints = {1, 2, 3}) - void rejectsAFileColumnBeforeFormatVersion4(int formatVersion) { - assertThatThrownBy(() -> newTableMetadata(formatVersion)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("Invalid type for photo: file is not supported until v4"); - } - - @Test - void acceptsAMetricsModeForAFileNestedField() { - TableMetadata metadata = - TableMetadata.newTableMetadata( - SCHEMA, - PartitionSpec.unpartitioned(), - "file:/tmp/table", - ImmutableMap.of( - TableProperties.FORMAT_VERSION, - "4", - TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", - "full")); - - assertThat(metadata.properties()) - .containsEntry(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", "full"); - } - - @Test - void replacementReservesDerivedIdsWhenAColumnBecomesAFile() { - TableMetadata metadata = - TableMetadata.newTableMetadata( - new Schema( - required(1, "id", Types.LongType.get()), - optional(2, "photo", Types.StringType.get()), - optional(3, "data", Types.StringType.get())), - PartitionSpec.unpartitioned(), - "file:/tmp/table", - ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); - Schema updated = - new Schema( - required(100, "id", Types.LongType.get()), - optional(101, "photo", Types.FileType.of(101)), - optional(108, "data", Types.StringType.get()), - optional(109, "notes", Types.StringType.get())); - Schema expected = - new Schema( - required(1, "id", Types.LongType.get()), - optional(4, "photo", Types.FileType.of(4)), - optional(3, "data", Types.StringType.get()), - optional(11, "notes", Types.StringType.get())); - - TableMetadata replacement = - metadata.buildReplacement( - updated, - PartitionSpec.unpartitioned(), - SortOrder.unsorted(), - metadata.location(), - ImmutableMap.of()); - - assertThat(replacement.schema().asStruct()).isEqualTo(expected.asStruct()); - assertThat(replacement.lastColumnId()).isEqualTo(expected.highestFieldId()); - } - - private static TableMetadata newTableMetadata(int formatVersion) { - return TableMetadata.newTableMetadata( - SCHEMA, - PartitionSpec.unpartitioned(), - "file:/tmp/table", - ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion))); - } -} diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java index c7812b446222..0130d62edb91 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java +++ b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java @@ -45,6 +45,17 @@ public class TestMetricsConfig { optional(CATEGORY, "category", Types.StringType.get()), optional(DATA, "data", Types.StringType.get())); + private static final Schema FILE_SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + private static final MetricsModes.MetricsMode NONE = MetricsModes.None.get(); + private static final MetricsModes.MetricsMode COUNTS = MetricsModes.Counts.get(); + private static final MetricsModes.MetricsMode TRUNCATE_16 = MetricsModes.Truncate.withLength(16); + private static final MetricsModes.MetricsMode FULL = MetricsModes.Full.get(); + @Test public void testInvalidColumnModeValue() { Map properties = @@ -220,6 +231,109 @@ public void testMetricsConfigNestedTypesStructs() { assertThat(config.metricsFieldIds()).containsExactly(33, 4); } + @Test + void fileFieldsWithoutPruningValueDefaultToCounts() { + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", COUNTS) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(ImmutableMap.of(), FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void fileColumnUsesInferredDefaultsOnlyForFilterableFields() { + Map properties = + Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "5"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", NONE) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", NONE) + .put("photo.inline", NONE) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void fileFieldsHaveNoMetricsPastTheInferredLimit() { + Map properties = + Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "1"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", NONE) + .put("photo.offset", NONE) + .put("photo.size", NONE) + .put("photo.content_type", NONE) + .put("photo.checksum", NONE) + .put("photo.inline", NONE) + .put("data", NONE) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void configuredModeOverridesAnInferredFileFieldDefault() { + Map properties = + Map.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", "full"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", FULL) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", COUNTS) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void configuredModeOverridesACountsFileFieldDefault() { + Map properties = + Map.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.checksum", "full"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", FULL) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + @Test void configCannotDisablePartitionSourceMetrics() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("category").build(); @@ -539,4 +653,10 @@ public void testMetricsConfigJavaSerialization() throws Exception { assertThat(deserialized.metricsFieldIds()).containsExactlyElementsOf(config.metricsFieldIds()); } + + private static Map leafColumnModes( + MetricsConfig config, Schema schema) { + return SchemaOrder.leafFieldIds(schema).stream() + .collect(Collectors.toMap(schema::findColumnName, config::columnMode)); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java index 2f319dd678f1..8ef9b58f9012 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java @@ -22,6 +22,9 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -71,6 +74,24 @@ public void mapOrder() { assertThat(SchemaOrder.leafFieldIds(schema)).containsExactly(1, 3, 4, 5); } + @Test + void fileOrder() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + List nestedIds = + Lists.transform(Types.FileType.of(2).fields(), Types.NestedField::fieldId); + + assertThat(SchemaOrder.allFieldIds(schema)) + .containsExactlyElementsOf( + ImmutableList.builder().add(1, 2).addAll(nestedIds).add(9).build()); + assertThat(SchemaOrder.leafFieldIds(schema)) + .containsExactlyElementsOf( + ImmutableList.builder().add(1).addAll(nestedIds).add(9).build()); + } + @Test public void mixedTypeOrder() { Schema schema = diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index d95a4925449b..ad3338be1676 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -83,6 +83,16 @@ public class TestSchemaUpdate { private static final int SCHEMA_LAST_COLUMN_ID = 23; + private static final Schema FILE_SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + private static SchemaUpdate fileUpdate() { + return new SchemaUpdate(FILE_SCHEMA, FILE_SCHEMA.highestFieldId()); + } + @Test public void testNoChanges() { Schema identical = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID).apply(); @@ -2591,16 +2601,6 @@ public void testCaseInsensitiveMoveAfterNewlyAddedField() { assertThat(actual.asStruct()).isEqualTo(expected.asStruct()); } - private static final Schema FILE_SCHEMA = - new Schema( - required(1, "id", Types.LongType.get()), - optional(2, "photo", Types.FileType.of(2)), - optional(9, "data", Types.StringType.get())); - - private static SchemaUpdate fileUpdate() { - return new SchemaUpdate(FILE_SCHEMA, FILE_SCHEMA.highestFieldId()); - } - @Test void cannotAddColumnToFileColumn() { assertThatThrownBy(() -> fileUpdate().addColumn("photo", "extra", Types.StringType.get())) @@ -2645,23 +2645,35 @@ void cannotUpdateFileNestedFieldDefault() { } @Test - void cannotUpdateFileNestedFieldRequirement() { + void cannotRequireFileNestedField() { assertThatThrownBy(() -> fileUpdate().requireColumn("photo.uri")) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotMakeFileNestedFieldOptional() { assertThatThrownBy(() -> fileUpdate().makeColumnOptional("photo.uri")) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot change a nested field of a file column: photo.uri"); } @Test - void cannotMoveFileNestedField() { + void cannotMoveFileNestedFieldFirst() { assertThatThrownBy(() -> fileUpdate().moveFirst("photo.checksum")) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void cannotMoveFileNestedFieldBefore() { assertThatThrownBy(() -> fileUpdate().moveBefore("photo.checksum", "photo.uri")) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void cannotMoveFileNestedFieldAfter() { assertThatThrownBy(() -> fileUpdate().moveAfter("photo.uri", "photo.inline")) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot move fields in non-struct type: file"); @@ -2669,17 +2681,21 @@ void cannotMoveFileNestedField() { @Test void renameFileColumn() { + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "image", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + Schema renamed = fileUpdate().renameColumn("photo", "image").apply(); - assertThat(renamed.findField("image").type()).isEqualTo(Types.FileType.of(2)); - assertThat(renamed.findField("image.uri").fieldId()).isEqualTo(3); + assertThat(renamed.asStruct()).isEqualTo(expected.asStruct()); } @Test void deleteFileColumn() { Schema deleted = fileUpdate().deleteColumn("photo").apply(); - assertThat(deleted.findField("photo")).isNull(); assertThat(deleted.asStruct()) .isEqualTo( new Schema( @@ -2705,4 +2721,72 @@ void addFileColumnReservesNestedIds() { assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); } + + @Test + void addStructWithFileFieldReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional( + 2, + "media", + Types.StructType.of( + optional(3, "photo", Types.FileType.of(3)), + optional(10, "caption", Types.StringType.get()))), + optional(11, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn( + "media", + Types.StructType.of( + optional(0, "photo", Types.FileType.of(0)), + optional(1, "caption", Types.StringType.get()))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void addListOfFilesReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photos", Types.ListType.ofOptional(3, Types.FileType.of(3))), + optional(10, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn("photos", Types.ListType.ofOptional(0, Types.FileType.of(0))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void addMapOfFilesReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional( + 2, + "byName", + Types.MapType.ofOptional(3, 4, Types.StringType.get(), Types.FileType.of(4))), + optional(11, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn( + "byName", + Types.MapType.ofOptional(0, 1, Types.StringType.get(), Types.FileType.of(1))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index bdf94c87edd0..32e0999607f4 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -2342,4 +2342,59 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form assertThat(updated.currentSnapshot().timestampMillis()).isEqualTo(snapshotTs); assertThat(updated.lastUpdatedMillis()).isLessThan(snapshotTs); } + + @ParameterizedTest + @FieldSource("org.apache.iceberg.TestHelpers#V3_AND_BELOW") + void rejectsAFileColumnBeforeFormatVersion4(int formatVersion) { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.FileType.of(2))); + + assertThatThrownBy( + () -> + TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Invalid type for photo: file is not supported until v4"); + } + + @Test + void replacementReservesDerivedIdsWhenAColumnBecomesAFile() { + TableMetadata metadata = + TableMetadata.newTableMetadata( + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())), + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); + Schema updated = + new Schema( + Types.NestedField.required(100, "id", Types.LongType.get()), + Types.NestedField.optional(101, "photo", Types.FileType.of(101)), + Types.NestedField.optional(108, "data", Types.StringType.get()), + Types.NestedField.optional(109, "notes", Types.StringType.get())); + Schema expected = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(4, "photo", Types.FileType.of(4)), + Types.NestedField.optional(3, "data", Types.StringType.get()), + Types.NestedField.optional(11, "notes", Types.StringType.get())); + + TableMetadata replacement = + metadata.buildReplacement( + updated, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + metadata.location(), + ImmutableMap.of()); + + assertThat(replacement.schema().asStruct()).isEqualTo(expected.asStruct()); + assertThat(replacement.lastColumnId()).isEqualTo(expected.highestFieldId()); + } } From 6f2217bc3660ffc4012e2ce7f01b79d6afac1757 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Mon, 16 Nov 2026 15:15:41 -0600 Subject: [PATCH 3/5] API, Core: Expose file nested field ID offsets Make the file type's nested field ID offsets public alongside NUM_NESTED_FIELDS, and keep the choice of which fields get inferred metrics private to MetricsConfig instead of on the type. Generated-by: Cursor --- .../java/org/apache/iceberg/types/Types.java | 27 +++++-------------- .../org/apache/iceberg/MetricsConfig.java | 7 ++++- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index 62df4c235c44..70b97498d863 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -1158,16 +1158,12 @@ private Map lazyFieldsById() { public static final class FileType extends NestedType { public static final String NAME = "file"; public static final int NUM_NESTED_FIELDS = 6; - - private static final int URI_ID_OFFSET = 1; - private static final int OFFSET_ID_OFFSET = 2; - private static final int SIZE_ID_OFFSET = 3; - private static final int CONTENT_TYPE_ID_OFFSET = 4; - private static final int CHECKSUM_ID_OFFSET = 5; - private static final int INLINE_ID_OFFSET = 6; - private static final int[] DEFAULT_INFERRED_METRIC_OFFSETS = { - URI_ID_OFFSET, SIZE_ID_OFFSET, CONTENT_TYPE_ID_OFFSET - }; + public static final int URI_ID_OFFSET = 1; + public static final int OFFSET_ID_OFFSET = 2; + public static final int SIZE_ID_OFFSET = 3; + public static final int CONTENT_TYPE_ID_OFFSET = 4; + public static final int CHECKSUM_ID_OFFSET = 5; + public static final int INLINE_ID_OFFSET = 6; private static final String URI = "uri"; private static final String OFFSET = "offset"; @@ -1191,17 +1187,6 @@ public static FileType of(int baseId) { return new FileType(baseId); } - /** - * Returns the offsets of nested fields that receive inferred metrics by default. - * - *

Offsets are relative to {@link #baseId()}. - * - * @return nested field offsets with inferred metrics - */ - public static int[] defaultInferredMetricOffsets() { - return DEFAULT_INFERRED_METRIC_OFFSETS; - } - private final int baseId; // Lazy and derived from baseId diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index 8625385cc9d9..dbe728b13f2a 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -50,6 +50,11 @@ public final class MetricsConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); + private static final int[] FILE_INFERRED_METRIC_ID_OFFSETS = { + Types.FileType.URI_ID_OFFSET, + Types.FileType.SIZE_ID_OFFSET, + Types.FileType.CONTENT_TYPE_ID_OFFSET + }; // Disable metrics by default for wide tables to prevent excessive metadata private static final MetricsMode DEFAULT_MODE = @@ -192,7 +197,7 @@ public Set struct(Types.StructType struct, Iterable> field @Override public Set file(Types.FileType file, Iterable> fieldResults) { - for (int offset : Types.FileType.defaultInferredMetricOffsets()) { + for (int offset : FILE_INFERRED_METRIC_ID_OFFSETS) { if (shouldContinue()) { idSet.add(file.baseId() + offset); } From fa6ec9ba7a4672ebbb6ff44271cc727212f25205 Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 18 Nov 2026 15:04:07 -0600 Subject: [PATCH 4/5] API, Core: Allow ID-preserving GetID functions with file columns A GetID that only implements get(int) now works on schemas with file columns as long as it keeps the file field's ID, so metadata table schemas that re-ID partition fields no longer fail for tables with file columns. Also simplifies ID reassignment tracking in Schema and renames a SchemaUpdate test to describe the rejected type update. Generated-by: Cursor --- .../main/java/org/apache/iceberg/Schema.java | 30 +++++++++---------- .../org/apache/iceberg/types/TypeUtil.java | 14 +++++---- .../apache/iceberg/types/TestFileType.java | 16 +++++++++- .../org/apache/iceberg/TestSchemaUpdate.java | 4 +-- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index 4b3909a3729e..a82ea27c5a1d 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -627,24 +627,24 @@ private List reassignIds(List columns, TypeUtil.GetID return columns; } - TypeUtil.GetID tracked = - new TypeUtil.GetID() { - @Override - public int get(int oldId) { - return track(oldId, getID.get(oldId)); - } - - @Override - public int get(int oldId, Type type) { - return track(oldId, getID.get(oldId, type)); - } - }; - - Type res = TypeUtil.assignIds(StructType.of(columns), tracked); + Type res = + TypeUtil.assignIds( + StructType.of(columns), + new TypeUtil.GetID() { + @Override + public int get(int oldId) { + return recordIfReassigned(oldId, getID.get(oldId)); + } + + @Override + public int get(int oldId, Type type) { + return recordIfReassigned(oldId, getID.get(oldId, type)); + } + }); return res.asStructType().fields(); } - private int track(int oldId, int newId) { + private int recordIfReassigned(int oldId, int newId) { if (newId != oldId) { idsToReassigned.put(oldId, newId); idsToOriginal.put(newId, oldId); diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 51e2908ef0cb..d6c79e52a60a 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -693,22 +693,24 @@ public interface GetID { /** * Assigns a new ID for a field of the given type. * - *

Implementations must override this method to assign IDs to file fields. + *

Implementations must override this method to assign a different ID to a file field. * * @param oldId an existing field ID * @param type the type of the field * @return a new field ID - * @throws UnsupportedOperationException if the type is a file and this method is not overridden + * @throws UnsupportedOperationException if the type is a file, this method is not overridden, + * and {@link #get(int)} returns a different ID */ default int get(int oldId, Type type) { - if (type.isFileType()) { + int newId = get(oldId); + if (type.isFileType() && newId != oldId) { throw new UnsupportedOperationException( String.format( - "Cannot assign IDs to file field %s: this GetID does not implement get(int, Type)", - oldId)); + "Cannot reassign file field %s to %s: this GetID does not implement get(int, Type)", + oldId, newId)); } - return get(oldId); + return newId; } } diff --git a/api/src/test/java/org/apache/iceberg/types/TestFileType.java b/api/src/test/java/org/apache/iceberg/types/TestFileType.java index 661ac5629070..4e75607d193d 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestFileType.java +++ b/api/src/test/java/org/apache/iceberg/types/TestFileType.java @@ -287,7 +287,21 @@ void assignedIdsRejectAnAssignerThatCannotReserve() { assertThatThrownBy(() -> TypeUtil.assignIds(schema.asStruct(), oldId -> oldId + 10)) .isInstanceOf(UnsupportedOperationException.class) .hasMessage( - "Cannot assign IDs to file field 1: this GetID does not implement get(int, Type)"); + "Cannot reassign file field 1 to 11: this GetID does not implement get(int, Type)"); + } + + @Test + void assignedIdsKeepAFileWhoseIdIsUnchanged() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + + Type assigned = TypeUtil.assignIds(schema.asStruct(), oldId -> oldId == 1 ? 100 : oldId); + + Types.StructType expected = + Types.StructType.of( + required(100, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + assertThat(assigned).isEqualTo(expected); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index ad3338be1676..a8b82e829661 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -2623,8 +2623,8 @@ void cannotRenameFileNestedField() { } @Test - void cannotPromoteFileNestedField() { - assertThatThrownBy(() -> fileUpdate().updateColumn("photo.size", Types.LongType.get())) + void cannotUpdateFileNestedFieldType() { + assertThatThrownBy(() -> fileUpdate().updateColumn("photo.size", Types.StringType.get())) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Cannot change a nested field of a file column: photo.size"); } From 9ed30e431c6930c09e8f817c8cbcaff3d5f7286c Mon Sep 17 00:00:00 2001 From: Russell Spitzer Date: Wed, 18 Nov 2026 15:07:15 -0600 Subject: [PATCH 5/5] API, Core: Keep file nested field ID offsets private MetricsConfig defines the offsets of the file fields that get inferred metrics itself, so FileType does not need to expose its ID layout. Generated-by: Cursor --- .../main/java/org/apache/iceberg/types/Types.java | 12 ++++++------ .../main/java/org/apache/iceberg/MetricsConfig.java | 7 ++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index 70b97498d863..387be83c32dc 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -1158,12 +1158,12 @@ private Map lazyFieldsById() { public static final class FileType extends NestedType { public static final String NAME = "file"; public static final int NUM_NESTED_FIELDS = 6; - public static final int URI_ID_OFFSET = 1; - public static final int OFFSET_ID_OFFSET = 2; - public static final int SIZE_ID_OFFSET = 3; - public static final int CONTENT_TYPE_ID_OFFSET = 4; - public static final int CHECKSUM_ID_OFFSET = 5; - public static final int INLINE_ID_OFFSET = 6; + private static final int URI_ID_OFFSET = 1; + private static final int OFFSET_ID_OFFSET = 2; + private static final int SIZE_ID_OFFSET = 3; + private static final int CONTENT_TYPE_ID_OFFSET = 4; + private static final int CHECKSUM_ID_OFFSET = 5; + private static final int INLINE_ID_OFFSET = 6; private static final String URI = "uri"; private static final String OFFSET = "offset"; diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index dbe728b13f2a..aa5ae9a66761 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -50,10 +50,11 @@ public final class MetricsConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); + private static final int FILE_URI_ID_OFFSET = 1; + private static final int FILE_SIZE_ID_OFFSET = 3; + private static final int FILE_CONTENT_TYPE_ID_OFFSET = 4; private static final int[] FILE_INFERRED_METRIC_ID_OFFSETS = { - Types.FileType.URI_ID_OFFSET, - Types.FileType.SIZE_ID_OFFSET, - Types.FileType.CONTENT_TYPE_ID_OFFSET + FILE_URI_ID_OFFSET, FILE_SIZE_ID_OFFSET, FILE_CONTENT_TYPE_ID_OFFSET }; // Disable metrics by default for wide tables to prevent excessive metadata