diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index eab56dd7ada3..a82ea27c5a1d 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -149,6 +149,8 @@ public Schema( this.struct = StructType.of(finalColumns); this.aliasToId = aliases != null ? ImmutableBiMap.copyOf(aliases) : null; + validateFileBaseIds(struct, null); + // validate IdentifierField if (identifierFieldIds != null) { Map idToParent = TypeUtil.indexParents(struct); @@ -161,6 +163,27 @@ public Schema( this.highestFieldId = lazyIdToName().keySet().stream().mapToInt(i -> i).max().orElse(0); } + private void validateFileBaseIds(Type type, String parentName) { + if (!type.isNestedType()) { + return; + } + + for (NestedField field : type.asNestedType().fields()) { + String name = parentName != null ? parentName + "." + field.name() : field.name(); + if (field.type().isFileType()) { + int baseId = field.type().asFileType().baseId(); + Preconditions.checkArgument( + baseId == field.fieldId(), + "Invalid file column %s: file base ID %s does not match field ID %s", + name, + baseId, + field.fieldId()); + } else { + validateFileBaseIds(field.type(), name); + } + } + } + static void validateIdentifierField( int fieldId, Map idToField, Map idToParent) { Types.NestedField field = idToField.get(fieldId); @@ -603,20 +626,33 @@ private List reassignIds(List columns, TypeUtil.GetID if (getID == null) { return columns; } + Type res = TypeUtil.assignIds( StructType.of(columns), - oldId -> { - int newId = getID.get(oldId); - if (newId != oldId) { - idsToReassigned.put(oldId, newId); - idsToOriginal.put(newId, oldId); + new TypeUtil.GetID() { + @Override + public int get(int oldId) { + return recordIfReassigned(oldId, getID.get(oldId)); + } + + @Override + public int get(int oldId, Type type) { + return recordIfReassigned(oldId, getID.get(oldId, type)); } - return newId; }); return res.asStructType().fields(); } + private int recordIfReassigned(int oldId, int newId) { + if (newId != oldId) { + idsToReassigned.put(oldId, newId); + idsToOriginal.put(newId, oldId); + } + + return newId; + } + /** * Check the compatibility of the schema with a format version. * diff --git a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java index 3b3a38ff5aeb..dc6f11c07e21 100644 --- a/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java +++ b/api/src/main/java/org/apache/iceberg/types/CheckCompatibility.java @@ -260,6 +260,15 @@ public List variant(Types.VariantType readVariant) { return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readVariant)); } + @Override + public List file(Types.FileType readFile, Iterable> fieldErrorLists) { + if (currentType.isFileType()) { + return NO_ERRORS; + } + + return ImmutableList.of(String.format(": %s cannot be read as a %s", currentType, readFile)); + } + @Override public List primitive(Type.PrimitiveType readPrimitive) { if (currentType.equals(readPrimitive)) { diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 51e2908ef0cb..d6c79e52a60a 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -693,22 +693,24 @@ public interface GetID { /** * Assigns a new ID for a field of the given type. * - *

Implementations must override this method to assign IDs to file fields. + *

Implementations must override this method to assign a different ID to a file field. * * @param oldId an existing field ID * @param type the type of the field * @return a new field ID - * @throws UnsupportedOperationException if the type is a file and this method is not overridden + * @throws UnsupportedOperationException if the type is a file, this method is not overridden, + * and {@link #get(int)} returns a different ID */ default int get(int oldId, Type type) { - if (type.isFileType()) { + int newId = get(oldId); + if (type.isFileType() && newId != oldId) { throw new UnsupportedOperationException( String.format( - "Cannot assign IDs to file field %s: this GetID does not implement get(int, Type)", - oldId)); + "Cannot reassign file field %s to %s: this GetID does not implement get(int, Type)", + oldId, newId)); } - return get(oldId); + return newId; } } diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index 3334fe38c41f..387be83c32dc 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -1158,6 +1158,12 @@ private Map lazyFieldsById() { public static final class FileType extends NestedType { public static final String NAME = "file"; public static final int NUM_NESTED_FIELDS = 6; + private static final int URI_ID_OFFSET = 1; + private static final int OFFSET_ID_OFFSET = 2; + private static final int SIZE_ID_OFFSET = 3; + private static final int CONTENT_TYPE_ID_OFFSET = 4; + private static final int CHECKSUM_ID_OFFSET = 5; + private static final int INLINE_ID_OFFSET = 6; private static final String URI = "uri"; private static final String OFFSET = "offset"; @@ -1284,12 +1290,13 @@ private List lazyFieldList() { if (fieldList == null) { this.fieldList = ImmutableList.of( - NestedField.optional(baseId + 1, URI, StringType.get()), - NestedField.optional(baseId + 2, OFFSET, LongType.get()), - NestedField.optional(baseId + 3, SIZE, LongType.get()), - NestedField.optional(baseId + 4, CONTENT_TYPE, StringType.get()), - NestedField.optional(baseId + 5, CHECKSUM, StringType.get()), - NestedField.optional(baseId + 6, INLINE, BinaryType.get())); + NestedField.optional(baseId + URI_ID_OFFSET, URI, StringType.get()), + NestedField.optional(baseId + OFFSET_ID_OFFSET, OFFSET, LongType.get()), + NestedField.optional(baseId + SIZE_ID_OFFSET, SIZE, LongType.get()), + NestedField.optional( + baseId + CONTENT_TYPE_ID_OFFSET, CONTENT_TYPE, StringType.get()), + NestedField.optional(baseId + CHECKSUM_ID_OFFSET, CHECKSUM, StringType.get()), + NestedField.optional(baseId + INLINE_ID_OFFSET, INLINE, BinaryType.get())); } return fieldList; diff --git a/api/src/test/java/org/apache/iceberg/TestSchema.java b/api/src/test/java/org/apache/iceberg/TestSchema.java index e410ee682991..16fc5b97af85 100644 --- a/api/src/test/java/org/apache/iceberg/TestSchema.java +++ b/api/src/test/java/org/apache/iceberg/TestSchema.java @@ -338,4 +338,51 @@ public void testIndexFieldsNestedSchema() { assertThat(fields.get(5).name()).isEqualTo("email"); assertThat(((Types.StructType) fields.get(2).type()).fields()).hasSize(3); } + + @Test + void rejectsASchemaWhereAnotherColumnHoldsADerivedId() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.FileType.of(2)), + Types.NestedField.optional(3, "data", Types.StringType.get()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContainingAll("3=photo.uri", "3=data"); + } + + @Test + void rejectsASchemaWhereTwoFileColumnsOverlap() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.optional(1, "photo", Types.FileType.of(1)), + Types.NestedField.optional(2, "thumbnail", Types.FileType.of(2)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Multiple entries with same key") + .hasMessageContaining("thumbnail"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIds() { + assertThatThrownBy( + () -> new Schema(Types.NestedField.optional(5, "photo", Types.FileType.of(9)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column photo: file base ID 9 does not match field ID 5"); + } + + @Test + void rejectsASchemaWithUnderivedNestedIdsInAMap() { + assertThatThrownBy( + () -> + new Schema( + Types.NestedField.optional( + 1, + "byName", + Types.MapType.ofOptional( + 2, 3, Types.StringType.get(), Types.FileType.of(9))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file column byName.value: file base ID 9 does not match field ID 3"); + } } diff --git a/api/src/test/java/org/apache/iceberg/types/TestFileType.java b/api/src/test/java/org/apache/iceberg/types/TestFileType.java index 661ac5629070..4e75607d193d 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestFileType.java +++ b/api/src/test/java/org/apache/iceberg/types/TestFileType.java @@ -287,7 +287,21 @@ void assignedIdsRejectAnAssignerThatCannotReserve() { assertThatThrownBy(() -> TypeUtil.assignIds(schema.asStruct(), oldId -> oldId + 10)) .isInstanceOf(UnsupportedOperationException.class) .hasMessage( - "Cannot assign IDs to file field 1: this GetID does not implement get(int, Type)"); + "Cannot reassign file field 1 to 11: this GetID does not implement get(int, Type)"); + } + + @Test + void assignedIdsKeepAFileWhoseIdIsUnchanged() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + + Type assigned = TypeUtil.assignIds(schema.asStruct(), oldId -> oldId == 1 ? 100 : oldId); + + Types.StructType expected = + Types.StructType.of( + required(100, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + assertThat(assigned).isEqualTo(expected); } @Test diff --git a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java index 20299cdafce2..e057680987d1 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java +++ b/api/src/test/java/org/apache/iceberg/types/TestReadabilityChecks.java @@ -61,6 +61,10 @@ public class TestReadabilityChecks { Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY), }; + private static final Schema FILE_SCHEMA = new Schema(optional(1, "photo", Types.FileType.of(1))); + private static final Schema FILE_AS_STRUCT_SCHEMA = + new Schema(optional(1, "photo", Types.StructType.of(Types.FileType.of(1).fields()))); + @Test public void testPrimitiveTypes() { for (Type.PrimitiveType from : PRIMITIVES) { @@ -286,6 +290,27 @@ public void testIncompatibleStructAndPrimitive() { .contains("struct cannot be read as a string"); } + @Test + void structCannotBeReadAsAFile() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_AS_STRUCT_SCHEMA)) + .singleElement() + .asString() + .endsWith("cannot be read as a file"); + } + + @Test + void fileCannotBeReadAsAStruct() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_AS_STRUCT_SCHEMA, FILE_SCHEMA)) + .singleElement() + .asString() + .endsWith("file cannot be read as a struct"); + } + + @Test + void fileCanBeReadAsAFile() { + assertThat(CheckCompatibility.readCompatibilityErrors(FILE_SCHEMA, FILE_SCHEMA)).isEmpty(); + } + @Test public void testMultipleErrors() { // required field is optional and cannot be promoted to the read type diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index f438c1bc5648..aa5ae9a66761 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -50,6 +50,12 @@ public final class MetricsConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); + private static final int FILE_URI_ID_OFFSET = 1; + private static final int FILE_SIZE_ID_OFFSET = 3; + private static final int FILE_CONTENT_TYPE_ID_OFFSET = 4; + private static final int[] FILE_INFERRED_METRIC_ID_OFFSETS = { + FILE_URI_ID_OFFSET, FILE_SIZE_ID_OFFSET, FILE_CONTENT_TYPE_ID_OFFSET + }; // Disable metrics by default for wide tables to prevent excessive metadata private static final MetricsMode DEFAULT_MODE = @@ -190,6 +196,17 @@ public Set struct(Types.StructType struct, Iterable> field return null; } + @Override + public Set file(Types.FileType file, Iterable> fieldResults) { + for (int offset : FILE_INFERRED_METRIC_ID_OFFSETS) { + if (shouldContinue()) { + idSet.add(file.baseId() + offset); + } + } + + return null; + } + @Override @SuppressWarnings("ReturnValueIgnored") public Set field(Types.NestedField field, Supplier> fieldResult) { @@ -275,6 +292,8 @@ private static MetricsConfig from( Map columnModes = Maps.newHashMap(); + columnModes.putAll(fileColumnModes(schema, defaultMode)); + columnModes.putAll(defaultColumnConf); // Default sort columns to at least truncate (overridden by config) @@ -313,6 +332,23 @@ private static Map defaultColumnModes( return builder.build(); } + private static Map fileColumnModes(Schema schema, MetricsMode defaultMode) { + ImmutableMap.Builder builder = ImmutableMap.builder(); + if (schema != null) { + MetricsMode fileMode = + defaultMode == MetricsModes.None.get() ? defaultMode : MetricsModes.Counts.get(); + for (Types.NestedField field : TypeUtil.indexById(schema.asStruct()).values()) { + if (field.type().isFileType()) { + for (Types.NestedField nested : field.type().asFileType().fields()) { + builder.put(schema.findColumnName(nested.fieldId()), fileMode); + } + } + } + } + + return builder.build(); + } + private static Map sortColumnModes( SortOrder order, MetricsMode configuredDefault) { ImmutableMap.Builder builder = ImmutableMap.builder(); diff --git a/core/src/main/java/org/apache/iceberg/SchemaOrder.java b/core/src/main/java/org/apache/iceberg/SchemaOrder.java index 4f5559fbae3e..730ade9721f6 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaOrder.java +++ b/core/src/main/java/org/apache/iceberg/SchemaOrder.java @@ -110,6 +110,11 @@ public List map( return null; } + @Override + public List file(Types.FileType file, Iterable> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public List variant(Types.VariantType variant) { return null; diff --git a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java index 0aed742f705f..c73979c7abc4 100644 --- a/core/src/main/java/org/apache/iceberg/SchemaUpdate.java +++ b/core/src/main/java/org/apache/iceberg/SchemaUpdate.java @@ -63,6 +63,7 @@ class SchemaUpdate implements UpdateSchema { private final Map addedNameToId = Maps.newHashMap(); private final Multimap moves = Multimaps.newListMultimap(Maps.newHashMap(), Lists::newArrayList); + private final TypeUtil.NextID nextColumnId = this::assignNewColumnId; private int lastColumnId; private boolean allowIncompatibleChanges = false; private Set identifierFieldNames; @@ -163,7 +164,7 @@ private void internalAddColumn( fullName); // assign new IDs in order - int newId = assignNewColumnId(); + int newId = nextColumnId.get(type); // update tracking for moves addedNameToId.put(caseSensitivityAwareName(fullName), newId); @@ -176,7 +177,7 @@ private void internalAddColumn( .withName(name) .isOptional(isOptional) .withId(newId) - .ofType(TypeUtil.assignFreshIds(type, this::assignNewColumnId)) + .ofType(typeForNewColumn(type, newId)) .withDoc(doc) .withInitialDefault(defaultValue) .withWriteDefault(defaultValue) @@ -186,10 +187,19 @@ private void internalAddColumn( parentToAddedIds.put(parentId, newId); } + private Type typeForNewColumn(Type type, int fieldId) { + if (type.isFileType()) { + return Types.FileType.of(fieldId); + } + + return TypeUtil.assignFreshIds(type, nextColumnId); + } + @Override public UpdateSchema deleteColumn(String name) { Types.NestedField field = findField(name); Preconditions.checkArgument(field != null, "Cannot delete missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !parentToAddedIds.containsKey(field.fieldId()), "Cannot delete a column that has additions: %s", @@ -205,6 +215,7 @@ public UpdateSchema deleteColumn(String name) { public UpdateSchema renameColumn(String name, String newName) { Types.NestedField field = findField(name); Preconditions.checkArgument(field != null, "Cannot rename missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument(newName != null, "Cannot rename a column to null"); Preconditions.checkArgument( !deletes.contains(field.fieldId()), @@ -241,6 +252,7 @@ public UpdateSchema makeColumnOptional(String name) { private void internalUpdateColumnRequirement(String name, boolean isOptional) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); if ((!isOptional && field.isRequired()) || (isOptional && field.isOptional())) { // if the change is a noop, allow it even if allowIncompatibleChanges is false @@ -273,6 +285,7 @@ private void internalUpdateColumnRequirement(String name, boolean isOptional) { public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -301,6 +314,7 @@ public UpdateSchema updateColumn(String name, Type.PrimitiveType newType) { public UpdateSchema updateColumnDoc(String name, String doc) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -322,6 +336,7 @@ public UpdateSchema updateColumnDoc(String name, String doc) { public UpdateSchema updateColumnDefault(String name, Literal newDefault) { Types.NestedField field = findForUpdate(name); Preconditions.checkArgument(field != null, "Cannot update missing column: %s", name); + checkNotNestedInFile(name, field.fieldId()); Preconditions.checkArgument( !deletes.contains(field.fieldId()), "Cannot update a column that will be deleted: %s", @@ -396,6 +411,15 @@ private boolean isAdded(String name) { return addedNameToId.containsKey(caseSensitivityAwareName(name)); } + private void checkNotNestedInFile(String name, int fieldId) { + Integer parentId = idToParent.get(fieldId); + Types.NestedField parent = parentId != null ? schema.findField(parentId) : null; + Preconditions.checkArgument( + parent == null || !parent.type().isFileType(), + "Cannot change a nested field of a file column: %s", + name); + } + private Types.NestedField findForUpdate(String name) { Types.NestedField existing = findField(name); if (existing != null) { @@ -655,6 +679,11 @@ public Type struct(Types.StructType struct, List fieldResults) { return struct; } + @Override + public Type file(Types.FileType file, List fieldResults) { + return file; + } + @Override public Type field(Types.NestedField field, Type fieldResult) { // the API validates deletes, updates, and additions don't conflict diff --git a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java index c7812b446222..0130d62edb91 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java +++ b/core/src/test/java/org/apache/iceberg/TestMetricsConfig.java @@ -45,6 +45,17 @@ public class TestMetricsConfig { optional(CATEGORY, "category", Types.StringType.get()), optional(DATA, "data", Types.StringType.get())); + private static final Schema FILE_SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + private static final MetricsModes.MetricsMode NONE = MetricsModes.None.get(); + private static final MetricsModes.MetricsMode COUNTS = MetricsModes.Counts.get(); + private static final MetricsModes.MetricsMode TRUNCATE_16 = MetricsModes.Truncate.withLength(16); + private static final MetricsModes.MetricsMode FULL = MetricsModes.Full.get(); + @Test public void testInvalidColumnModeValue() { Map properties = @@ -220,6 +231,109 @@ public void testMetricsConfigNestedTypesStructs() { assertThat(config.metricsFieldIds()).containsExactly(33, 4); } + @Test + void fileFieldsWithoutPruningValueDefaultToCounts() { + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", COUNTS) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(ImmutableMap.of(), FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void fileColumnUsesInferredDefaultsOnlyForFilterableFields() { + Map properties = + Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "5"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", NONE) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", NONE) + .put("photo.inline", NONE) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void fileFieldsHaveNoMetricsPastTheInferredLimit() { + Map properties = + Map.of(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "1"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", NONE) + .put("photo.offset", NONE) + .put("photo.size", NONE) + .put("photo.content_type", NONE) + .put("photo.checksum", NONE) + .put("photo.inline", NONE) + .put("data", NONE) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void configuredModeOverridesAnInferredFileFieldDefault() { + Map properties = + Map.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.size", "full"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", FULL) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", COUNTS) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + + @Test + void configuredModeOverridesACountsFileFieldDefault() { + Map properties = + Map.of(TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "photo.checksum", "full"); + Map expected = + ImmutableMap.builder() + .put("id", TRUNCATE_16) + .put("photo.uri", TRUNCATE_16) + .put("photo.offset", COUNTS) + .put("photo.size", TRUNCATE_16) + .put("photo.content_type", TRUNCATE_16) + .put("photo.checksum", FULL) + .put("photo.inline", COUNTS) + .put("data", TRUNCATE_16) + .buildOrThrow(); + + MetricsConfig config = MetricsTestUtil.from(properties, FILE_SCHEMA); + + assertThat(leafColumnModes(config, FILE_SCHEMA)).isEqualTo(expected); + } + @Test void configCannotDisablePartitionSourceMetrics() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).identity("category").build(); @@ -539,4 +653,10 @@ public void testMetricsConfigJavaSerialization() throws Exception { assertThat(deserialized.metricsFieldIds()).containsExactlyElementsOf(config.metricsFieldIds()); } + + private static Map leafColumnModes( + MetricsConfig config, Schema schema) { + return SchemaOrder.leafFieldIds(schema).stream() + .collect(Collectors.toMap(schema::findColumnName, config::columnMode)); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java index 2f319dd678f1..8ef9b58f9012 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaOrder.java @@ -22,6 +22,9 @@ import static org.apache.iceberg.types.Types.NestedField.required; import static org.assertj.core.api.Assertions.assertThat; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -71,6 +74,24 @@ public void mapOrder() { assertThat(SchemaOrder.leafFieldIds(schema)).containsExactly(1, 3, 4, 5); } + @Test + void fileOrder() { + Schema schema = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + List nestedIds = + Lists.transform(Types.FileType.of(2).fields(), Types.NestedField::fieldId); + + assertThat(SchemaOrder.allFieldIds(schema)) + .containsExactlyElementsOf( + ImmutableList.builder().add(1, 2).addAll(nestedIds).add(9).build()); + assertThat(SchemaOrder.leafFieldIds(schema)) + .containsExactlyElementsOf( + ImmutableList.builder().add(1).addAll(nestedIds).add(9).build()); + } + @Test public void mixedTypeOrder() { Schema schema = diff --git a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java index 5325e4013c68..a8b82e829661 100644 --- a/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java +++ b/core/src/test/java/org/apache/iceberg/TestSchemaUpdate.java @@ -83,6 +83,16 @@ public class TestSchemaUpdate { private static final int SCHEMA_LAST_COLUMN_ID = 23; + private static final Schema FILE_SCHEMA = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + private static SchemaUpdate fileUpdate() { + return new SchemaUpdate(FILE_SCHEMA, FILE_SCHEMA.highestFieldId()); + } + @Test public void testNoChanges() { Schema identical = new SchemaUpdate(SCHEMA, SCHEMA_LAST_COLUMN_ID).apply(); @@ -2590,4 +2600,193 @@ public void testCaseInsensitiveMoveAfterNewlyAddedField() { assertThat(actual.asStruct()).isEqualTo(expected.asStruct()); } + + @Test + void cannotAddColumnToFileColumn() { + assertThatThrownBy(() -> fileUpdate().addColumn("photo", "extra", Types.StringType.get())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot add to non-struct column: photo: file"); + } + + @Test + void cannotDeleteFileNestedField() { + assertThatThrownBy(() -> fileUpdate().deleteColumn("photo.checksum")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.checksum"); + } + + @Test + void cannotRenameFileNestedField() { + assertThatThrownBy(() -> fileUpdate().renameColumn("photo.uri", "location")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotUpdateFileNestedFieldType() { + assertThatThrownBy(() -> fileUpdate().updateColumn("photo.size", Types.StringType.get())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.size"); + } + + @Test + void cannotUpdateFileNestedFieldDoc() { + assertThatThrownBy(() -> fileUpdate().updateColumnDoc("photo.uri", "the location")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotUpdateFileNestedFieldDefault() { + assertThatThrownBy( + () -> fileUpdate().updateColumnDefault("photo.uri", Literal.of("s3://bucket/key"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotRequireFileNestedField() { + assertThatThrownBy(() -> fileUpdate().requireColumn("photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotMakeFileNestedFieldOptional() { + assertThatThrownBy(() -> fileUpdate().makeColumnOptional("photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot change a nested field of a file column: photo.uri"); + } + + @Test + void cannotMoveFileNestedFieldFirst() { + assertThatThrownBy(() -> fileUpdate().moveFirst("photo.checksum")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void cannotMoveFileNestedFieldBefore() { + assertThatThrownBy(() -> fileUpdate().moveBefore("photo.checksum", "photo.uri")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void cannotMoveFileNestedFieldAfter() { + assertThatThrownBy(() -> fileUpdate().moveAfter("photo.uri", "photo.inline")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot move fields in non-struct type: file"); + } + + @Test + void renameFileColumn() { + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "image", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema renamed = fileUpdate().renameColumn("photo", "image").apply(); + + assertThat(renamed.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void deleteFileColumn() { + Schema deleted = fileUpdate().deleteColumn("photo").apply(); + + assertThat(deleted.asStruct()) + .isEqualTo( + new Schema( + required(1, "id", Types.LongType.get()), + optional(9, "data", Types.StringType.get())) + .asStruct()); + } + + @Test + void addFileColumnReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn("photo", Types.FileType.of(0)) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void addStructWithFileFieldReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional( + 2, + "media", + Types.StructType.of( + optional(3, "photo", Types.FileType.of(3)), + optional(10, "caption", Types.StringType.get()))), + optional(11, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn( + "media", + Types.StructType.of( + optional(0, "photo", Types.FileType.of(0)), + optional(1, "caption", Types.StringType.get()))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void addListOfFilesReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photos", Types.ListType.ofOptional(3, Types.FileType.of(3))), + optional(10, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn("photos", Types.ListType.ofOptional(0, Types.FileType.of(0))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void addMapOfFilesReservesNestedIds() { + Schema schema = new Schema(required(1, "id", Types.LongType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional( + 2, + "byName", + Types.MapType.ofOptional(3, 4, Types.StringType.get(), Types.FileType.of(4))), + optional(11, "data", Types.StringType.get())); + + Schema updated = + new SchemaUpdate(schema, schema.highestFieldId()) + .addColumn( + "byName", + Types.MapType.ofOptional(0, 1, Types.StringType.get(), Types.FileType.of(1))) + .addColumn("data", Types.StringType.get()) + .apply(); + + assertThat(updated.asStruct()).isEqualTo(expected.asStruct()); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index bdf94c87edd0..32e0999607f4 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -2342,4 +2342,59 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form assertThat(updated.currentSnapshot().timestampMillis()).isEqualTo(snapshotTs); assertThat(updated.lastUpdatedMillis()).isLessThan(snapshotTs); } + + @ParameterizedTest + @FieldSource("org.apache.iceberg.TestHelpers#V3_AND_BELOW") + void rejectsAFileColumnBeforeFormatVersion4(int formatVersion) { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.FileType.of(2))); + + assertThatThrownBy( + () -> + TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, String.valueOf(formatVersion)))) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("Invalid type for photo: file is not supported until v4"); + } + + @Test + void replacementReservesDerivedIdsWhenAColumnBecomesAFile() { + TableMetadata metadata = + TableMetadata.newTableMetadata( + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "photo", Types.StringType.get()), + Types.NestedField.optional(3, "data", Types.StringType.get())), + PartitionSpec.unpartitioned(), + "file:/tmp/table", + ImmutableMap.of(TableProperties.FORMAT_VERSION, "4")); + Schema updated = + new Schema( + Types.NestedField.required(100, "id", Types.LongType.get()), + Types.NestedField.optional(101, "photo", Types.FileType.of(101)), + Types.NestedField.optional(108, "data", Types.StringType.get()), + Types.NestedField.optional(109, "notes", Types.StringType.get())); + Schema expected = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(4, "photo", Types.FileType.of(4)), + Types.NestedField.optional(3, "data", Types.StringType.get()), + Types.NestedField.optional(11, "notes", Types.StringType.get())); + + TableMetadata replacement = + metadata.buildReplacement( + updated, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + metadata.location(), + ImmutableMap.of()); + + assertThat(replacement.schema().asStruct()).isEqualTo(expected.asStruct()); + assertThat(replacement.lastColumnId()).isEqualTo(expected.highestFieldId()); + } }