diff --git a/api/src/main/java/org/apache/iceberg/Accessors.java b/api/src/main/java/org/apache/iceberg/Accessors.java index 0b36730fbb4b..b285722ae883 100644 --- a/api/src/main/java/org/apache/iceberg/Accessors.java +++ b/api/src/main/java/org/apache/iceberg/Accessors.java @@ -232,6 +232,12 @@ public Map> struct( return accessors; } + @Override + public Map> file( + Types.FileType file, List>> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public Map> variant(Types.VariantType variant) { return null; diff --git a/api/src/main/java/org/apache/iceberg/Schema.java b/api/src/main/java/org/apache/iceberg/Schema.java index 3e59998be476..f63a5f9a2604 100644 --- a/api/src/main/java/org/apache/iceberg/Schema.java +++ b/api/src/main/java/org/apache/iceberg/Schema.java @@ -67,7 +67,8 @@ public class Schema implements Serializable { Type.TypeID.VARIANT, 3, Type.TypeID.UNKNOWN, 3, Type.TypeID.GEOMETRY, 3, - Type.TypeID.GEOGRAPHY, 3); + Type.TypeID.GEOGRAPHY, 3, + Type.TypeID.FILE, 4); private final StructType struct; private final int schemaId; diff --git a/api/src/main/java/org/apache/iceberg/types/AssignFreshIds.java b/api/src/main/java/org/apache/iceberg/types/AssignFreshIds.java index f3759f1d72f3..27ac5b47c672 100644 --- a/api/src/main/java/org/apache/iceberg/types/AssignFreshIds.java +++ b/api/src/main/java/org/apache/iceberg/types/AssignFreshIds.java @@ -48,15 +48,18 @@ class AssignFreshIds extends TypeUtil.CustomOrderSchemaVisitor { this.nextId = nextId; } - private int idFor(String fullName) { + private int idFor(int currentId, Type type) { + String fullName = name(currentId); if (baseSchema != null && fullName != null) { - Types.NestedField field = baseSchema.findField(fullName); - if (field != null) { - return field.fieldId(); + Types.NestedField baseField = baseSchema.findField(fullName); + // a file derives its nested field IDs from its own ID, so reusing a base ID is only safe when + // the base field was already a file and had reserved that block + if (baseField != null && (!type.isFileType() || baseField.type().isFileType())) { + return baseField.fieldId(); } } - return nextId.get(); + return nextId.get(type); } private String name(int id) { @@ -80,15 +83,17 @@ public Type struct(Types.StructType struct, Iterable futures) { // assign IDs for this struct's fields first List newIds = Lists.newArrayListWithExpectedSize(length); for (int i = 0; i < length; i += 1) { - newIds.add(idFor(name(fields.get(i).fieldId()))); + Types.NestedField field = fields.get(i); + newIds.add(idFor(field.fieldId(), field.type())); } List newFields = Lists.newArrayListWithExpectedSize(length); Iterator types = futures.iterator(); for (int i = 0; i < length; i += 1) { Types.NestedField field = fields.get(i); - Type type = types.next(); - newFields.add(Types.NestedField.from(field).withId(newIds.get(i)).ofType(type).build()); + int newId = newIds.get(i); + Type type = TypeUtil.applyAssignedId(newId, types.next()); + newFields.add(Types.NestedField.from(field).withId(newId).ofType(type).build()); } return Types.StructType.of(newFields); @@ -101,22 +106,25 @@ public Type field(Types.NestedField field, Supplier future) { @Override public Type list(Types.ListType list, Supplier future) { - int newId = idFor(name(list.elementId())); + int newId = idFor(list.elementId(), list.elementType()); + Type elementType = TypeUtil.applyAssignedId(newId, future.get()); if (list.isElementOptional()) { - return Types.ListType.ofOptional(newId, future.get()); + return Types.ListType.ofOptional(newId, elementType); } else { - return Types.ListType.ofRequired(newId, future.get()); + return Types.ListType.ofRequired(newId, elementType); } } @Override public Type map(Types.MapType map, Supplier keyFuture, Supplier valueFuture) { - int newKeyId = idFor(name(map.keyId())); - int newValueId = idFor(name(map.valueId())); + int newKeyId = idFor(map.keyId(), map.keyType()); + int newValueId = idFor(map.valueId(), map.valueType()); + Type keyType = TypeUtil.applyAssignedId(newKeyId, keyFuture.get()); + Type valueType = TypeUtil.applyAssignedId(newValueId, valueFuture.get()); if (map.isValueOptional()) { - return Types.MapType.ofOptional(newKeyId, newValueId, keyFuture.get(), valueFuture.get()); + return Types.MapType.ofOptional(newKeyId, newValueId, keyType, valueType); } else { - return Types.MapType.ofRequired(newKeyId, newValueId, keyFuture.get(), valueFuture.get()); + return Types.MapType.ofRequired(newKeyId, newValueId, keyType, valueType); } } @@ -125,6 +133,12 @@ public Type variant(Types.VariantType variant) { return variant; } + @Override + public Type file(Types.FileType file, Iterable futures) { + // nested fields are rebuilt from the new id assigned to the field that holds this type + return file; + } + @Override public Type primitive(Type.PrimitiveType primitive) { return primitive; diff --git a/api/src/main/java/org/apache/iceberg/types/AssignIds.java b/api/src/main/java/org/apache/iceberg/types/AssignIds.java index fd5ac7ff67b9..f1f5b5d2034c 100644 --- a/api/src/main/java/org/apache/iceberg/types/AssignIds.java +++ b/api/src/main/java/org/apache/iceberg/types/AssignIds.java @@ -31,8 +31,8 @@ class AssignIds extends TypeUtil.CustomOrderSchemaVisitor { this.getID = getID; } - private int idFor(int id) { - return getID.get(id); + private int idFor(int id, Type type) { + return getID.get(id, type); } @Override @@ -48,15 +48,16 @@ public Type struct(Types.StructType struct, Iterable futures) { // assign IDs for this struct's fields first List newIds = Lists.newArrayListWithExpectedSize(length); for (Types.NestedField field : fields) { - newIds.add(idFor(field.fieldId())); + newIds.add(idFor(field.fieldId(), field.type())); } List newFields = Lists.newArrayListWithExpectedSize(length); Iterator types = futures.iterator(); for (int i = 0; i < length; i += 1) { Types.NestedField field = fields.get(i); - Type type = types.next(); - newFields.add(Types.NestedField.from(field).withId(newIds.get(i)).ofType(type).build()); + int newId = newIds.get(i); + Type type = TypeUtil.applyAssignedId(newId, types.next()); + newFields.add(Types.NestedField.from(field).withId(newId).ofType(type).build()); } return Types.StructType.of(newFields); @@ -69,22 +70,25 @@ public Type field(Types.NestedField field, Supplier future) { @Override public Type list(Types.ListType list, Supplier future) { - int newId = idFor(list.elementId()); + int newId = idFor(list.elementId(), list.elementType()); + Type elementType = TypeUtil.applyAssignedId(newId, future.get()); if (list.isElementOptional()) { - return Types.ListType.ofOptional(newId, future.get()); + return Types.ListType.ofOptional(newId, elementType); } else { - return Types.ListType.ofRequired(newId, future.get()); + return Types.ListType.ofRequired(newId, elementType); } } @Override public Type map(Types.MapType map, Supplier keyFuture, Supplier valueFuture) { - int newKeyId = idFor(map.keyId()); - int newValueId = idFor(map.valueId()); + int newKeyId = idFor(map.keyId(), map.keyType()); + int newValueId = idFor(map.valueId(), map.valueType()); + Type keyType = TypeUtil.applyAssignedId(newKeyId, keyFuture.get()); + Type valueType = TypeUtil.applyAssignedId(newValueId, valueFuture.get()); if (map.isValueOptional()) { - return Types.MapType.ofOptional(newKeyId, newValueId, keyFuture.get(), valueFuture.get()); + return Types.MapType.ofOptional(newKeyId, newValueId, keyType, valueType); } else { - return Types.MapType.ofRequired(newKeyId, newValueId, keyFuture.get(), valueFuture.get()); + return Types.MapType.ofRequired(newKeyId, newValueId, keyType, valueType); } } @@ -93,6 +97,12 @@ public Type variant(Types.VariantType variant) { return variant; } + @Override + public Type file(Types.FileType file, Iterable futures) { + // nested fields are rebuilt from the new id assigned to the field that holds this type + return file; + } + @Override public Type primitive(Type.PrimitiveType primitive) { return primitive; diff --git a/api/src/main/java/org/apache/iceberg/types/GetProjectedIds.java b/api/src/main/java/org/apache/iceberg/types/GetProjectedIds.java index 1ec70b8578bc..81ed98b6bacf 100644 --- a/api/src/main/java/org/apache/iceberg/types/GetProjectedIds.java +++ b/api/src/main/java/org/apache/iceberg/types/GetProjectedIds.java @@ -47,7 +47,7 @@ public Set struct(Types.StructType struct, List> fieldResu @Override public Set field(Types.NestedField field, Set fieldResult) { - if ((includeStructIds && field.type().isStructType()) + if ((includeStructIds && (field.type().isStructType() || field.type().isFileType())) || field.type().isPrimitiveType() || field.type().isVariantType()) { fieldIds.add(field.fieldId()); @@ -75,6 +75,11 @@ public Set map(Types.MapType map, Set keyResult, Set return fieldIds; } + @Override + public Set file(Types.FileType file, List> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public Set variant(Types.VariantType variant) { return null; diff --git a/api/src/main/java/org/apache/iceberg/types/IndexById.java b/api/src/main/java/org/apache/iceberg/types/IndexById.java index a7b96eb381f7..56b7e6f0da94 100644 --- a/api/src/main/java/org/apache/iceberg/types/IndexById.java +++ b/api/src/main/java/org/apache/iceberg/types/IndexById.java @@ -65,6 +65,12 @@ public Map map( return null; } + @Override + public Map file( + Types.FileType file, List> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public Map variant(Types.VariantType variant) { return null; diff --git a/api/src/main/java/org/apache/iceberg/types/IndexByName.java b/api/src/main/java/org/apache/iceberg/types/IndexByName.java index 9ca2a1d3396c..115099011165 100644 --- a/api/src/main/java/org/apache/iceberg/types/IndexByName.java +++ b/api/src/main/java/org/apache/iceberg/types/IndexByName.java @@ -189,6 +189,11 @@ public Map map( return nameToId; } + @Override + public Map file(Types.FileType file, List> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public Map variant(Types.VariantType variant) { return nameToId; diff --git a/api/src/main/java/org/apache/iceberg/types/IndexParents.java b/api/src/main/java/org/apache/iceberg/types/IndexParents.java index 6e611d47e912..1c3ab2696432 100644 --- a/api/src/main/java/org/apache/iceberg/types/IndexParents.java +++ b/api/src/main/java/org/apache/iceberg/types/IndexParents.java @@ -76,6 +76,11 @@ public Map map( return idToParent; } + @Override + public Map file(Types.FileType file, List> fieldResults) { + return struct(file.asStruct(), fieldResults); + } + @Override public Map variant(Types.VariantType variant) { return idToParent; diff --git a/api/src/main/java/org/apache/iceberg/types/ReassignDoc.java b/api/src/main/java/org/apache/iceberg/types/ReassignDoc.java index 86527fb3897f..de63e94ffc6b 100644 --- a/api/src/main/java/org/apache/iceberg/types/ReassignDoc.java +++ b/api/src/main/java/org/apache/iceberg/types/ReassignDoc.java @@ -96,6 +96,12 @@ public Type variant(Types.VariantType variant) { return variant; } + @Override + public Type file(Types.FileType file, Iterable fieldTypes) { + // the nested fields of a file cannot carry docs + return file; + } + @Override public Type primitive(Type.PrimitiveType primitive) { return primitive; diff --git a/api/src/main/java/org/apache/iceberg/types/ReassignIds.java b/api/src/main/java/org/apache/iceberg/types/ReassignIds.java index 3d114f093f6b..611c738c741f 100644 --- a/api/src/main/java/org/apache/iceberg/types/ReassignIds.java +++ b/api/src/main/java/org/apache/iceberg/types/ReassignIds.java @@ -50,7 +50,7 @@ public Type schema(Schema schema, Supplier future) { } } - private int id(Types.StructType sourceStruct, String name) { + private int id(Types.StructType sourceStruct, String name, Type type) { Types.NestedField sourceField = caseSensitive ? sourceStruct.field(name) : sourceStruct.caseInsensitiveField(name); @@ -59,7 +59,7 @@ private int id(Types.StructType sourceStruct, String name) { } if (assignId != null) { - return assignId.get(); + return assignId.get(type); } throw new IllegalArgumentException("Field " + name + " not found in source schema"); @@ -78,8 +78,9 @@ public Type struct(Types.StructType struct, Iterable fieldTypes) { List newFields = Lists.newArrayListWithExpectedSize(length); for (int i = 0; i < length; i += 1) { Types.NestedField field = fields.get(i); - int fieldId = id(sourceStruct, field.name()); - newFields.add(Types.NestedField.from(field).withId(fieldId).ofType(types.get(i)).build()); + int fieldId = id(sourceStruct, field.name(), field.type()); + Type type = TypeUtil.applyAssignedId(fieldId, types.get(i)); + newFields.add(Types.NestedField.from(field).withId(fieldId).ofType(type).build()); } return Types.StructType.of(newFields); @@ -120,10 +121,11 @@ public Type list(Types.ListType list, Supplier elementTypeFuture) { this.sourceType = sourceList.elementType(); try { + Type elementType = TypeUtil.applyAssignedId(sourceElementId, elementTypeFuture.get()); if (list.isElementOptional()) { - return Types.ListType.ofOptional(sourceElementId, elementTypeFuture.get()); + return Types.ListType.ofOptional(sourceElementId, elementType); } else { - return Types.ListType.ofRequired(sourceElementId, elementTypeFuture.get()); + return Types.ListType.ofRequired(sourceElementId, elementType); } } finally { @@ -141,10 +143,10 @@ public Type map(Types.MapType map, Supplier keyTypeFuture, Supplier try { this.sourceType = sourceMap.keyType(); - Type keyType = keyTypeFuture.get(); + Type keyType = TypeUtil.applyAssignedId(sourceKeyId, keyTypeFuture.get()); this.sourceType = sourceMap.valueType(); - Type valueType = valueTypeFuture.get(); + Type valueType = TypeUtil.applyAssignedId(sourceValueId, valueTypeFuture.get()); if (map.isValueOptional()) { return Types.MapType.ofOptional(sourceKeyId, sourceValueId, keyType, valueType); @@ -162,6 +164,12 @@ public Type variant(Types.VariantType variant) { return variant; } + @Override + public Type file(Types.FileType file, Iterable fieldTypes) { + // nested fields are rebuilt from the id assigned to the field that holds this type + return file; + } + @Override public Type primitive(Type.PrimitiveType primitive) { return primitive; // nothing to reassign diff --git a/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java b/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java index 1c94bd57c114..9e767e8babce 100644 --- a/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java +++ b/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java @@ -37,7 +37,16 @@ public Type schema(Schema schema, Type structResult) { @Override public Type struct(Types.StructType struct, List fieldResults) { - List fields = struct.fields(); + return replaceFieldTypes(struct.fields(), fieldResults, struct); + } + + @Override + public Type file(Types.FileType file, List fieldResults) { + return replaceFieldTypes(file.fields(), fieldResults, file); + } + + private Type replaceFieldTypes( + List fields, List fieldResults, Type unchangedResult) { List newFields = Lists.newArrayListWithExpectedSize(fields.size()); boolean hasChanged = false; @@ -56,7 +65,7 @@ public Type struct(Types.StructType struct, List fieldResults) { return Types.StructType.of(newFields); } - return struct; + return unchangedResult; } @Override diff --git a/api/src/main/java/org/apache/iceberg/types/Type.java b/api/src/main/java/org/apache/iceberg/types/Type.java index bed478d938e7..d27305f7ead3 100644 --- a/api/src/main/java/org/apache/iceberg/types/Type.java +++ b/api/src/main/java/org/apache/iceberg/types/Type.java @@ -50,6 +50,7 @@ enum TypeID { LIST(List.class), MAP(Map.class), VARIANT(Variant.class), + FILE(StructLike.class), UNKNOWN(Object.class); private final Class javaClass; @@ -89,6 +90,10 @@ default Types.VariantType asVariantType() { throw new IllegalArgumentException("Not a variant type: " + this); } + default Types.FileType asFileType() { + throw new IllegalArgumentException("Not a file type: " + this); + } + default boolean isNestedType() { return false; } @@ -109,6 +114,10 @@ default boolean isVariantType() { return false; } + default boolean isFileType() { + return false; + } + default NestedType asNestedType() { throw new IllegalArgumentException("Not a nested type: " + this); } diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index 8e39ae7a43bc..51e2908ef0cb 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -460,6 +460,34 @@ public static Type assignIds(Type type, GetID getId) { return TypeUtil.visit(type, new AssignIds(getId)); } + /** + * Returns the type to use for a field that was assigned {@code newId}. + * + *

A file's nested field IDs are derived from the ID of the field that holds it, so a file is + * rebuilt from the new ID rather than reassigned field by field. + * + * @param newId the ID assigned to the field + * @param assignedType the field's type with IDs assigned + * @return the type the field should have + */ + static Type applyAssignedId(int newId, Type assignedType) { + if (assignedType.isFileType()) { + return Types.FileType.of(newId); + } + + return assignedType; + } + + /** + * Returns the number of IDs that must be reserved after the ID of a field with the given type. + * + * @param type a field type + * @return the number of consecutive IDs following the field's ID that the type derives + */ + private static int numReservedIds(Type type) { + return type.isFileType() ? Types.FileType.NUM_NESTED_FIELDS : 0; + } + public static Type find(Schema schema, Predicate predicate) { return visit(schema, new FindTypeVisitor(predicate)); } @@ -639,11 +667,49 @@ private static int estimateSize(Type type) { /** Interface for passing a function that assigns column IDs. */ public interface NextID { int get(); + + default int get(Type type) { + int numReserved = numReservedIds(type); + int id = get(); + for (int offset = 1; offset <= numReserved; offset += 1) { + int reserved = get(); + Preconditions.checkState( + reserved == id + offset, + "Failed to assign consecutive IDs %s-%s, requested %s and got %s", + id + 1, + id + numReserved, + id + offset, + reserved); + } + + return id; + } } /** Interface for passing a function that assigns column IDs from the previous Id. */ public interface GetID { int get(int oldId); + + /** + * Assigns a new ID for a field of the given type. + * + *

Implementations must override this method to assign IDs to file fields. + * + * @param oldId an existing field ID + * @param type the type of the field + * @return a new field ID + * @throws UnsupportedOperationException if the type is a file and this method is not overridden + */ + default int get(int oldId, Type type) { + if (type.isFileType()) { + throw new UnsupportedOperationException( + String.format( + "Cannot assign IDs to file field %s: this GetID does not implement get(int, Type)", + oldId)); + } + + return get(oldId); + } } /** @@ -753,6 +819,10 @@ public T variant(Types.VariantType variant) { throw new UnsupportedOperationException("Unsupported type: variant"); } + public T file(Types.FileType file, List fieldResults) { + throw new UnsupportedOperationException("Unsupported type: file"); + } + public T primitive(Type.PrimitiveType primitive) { return null; } @@ -766,18 +836,11 @@ public static T visit(Type type, SchemaVisitor visitor) { switch (type.typeId()) { case STRUCT: Types.StructType struct = type.asNestedType().asStructType(); - List results = Lists.newArrayListWithExpectedSize(struct.fields().size()); - for (Types.NestedField field : struct.fields()) { - visitor.beforeField(field); - T result; - try { - result = visit(field.type(), visitor); - } finally { - visitor.afterField(field); - } - results.add(visitor.field(field, result)); - } - return visitor.struct(struct, results); + return visitor.struct(struct, visitFields(struct.fields(), visitor)); + + case FILE: + Types.FileType file = type.asFileType(); + return visitor.file(file, visitFields(file.fields(), visitor)); case LIST: Types.ListType list = type.asNestedType().asListType(); @@ -824,6 +887,21 @@ public static T visit(Type type, SchemaVisitor visitor) { } } + private static List visitFields(List fields, SchemaVisitor visitor) { + List results = Lists.newArrayListWithExpectedSize(fields.size()); + for (Types.NestedField field : fields) { + visitor.beforeField(field); + T result; + try { + result = visit(field.type(), visitor); + } finally { + visitor.afterField(field); + } + results.add(visitor.field(field, result)); + } + return results; + } + public static class CustomOrderSchemaVisitor { public T schema(Schema schema, Supplier structResult) { return null; @@ -849,6 +927,10 @@ public T variant(Types.VariantType variant) { throw new UnsupportedOperationException("Unsupported type: variant"); } + public T file(Types.FileType file, Iterable fieldResults) { + throw new UnsupportedOperationException("Unsupported type: file"); + } + public T primitive(Type.PrimitiveType primitive) { return null; } @@ -906,13 +988,11 @@ public static T visit(Type type, CustomOrderSchemaVisitor visitor) { switch (type.typeId()) { case STRUCT: Types.StructType struct = type.asNestedType().asStructType(); - List> results = - Lists.newArrayListWithExpectedSize(struct.fields().size()); - for (Types.NestedField field : struct.fields()) { - results.add(new VisitFieldFuture<>(field, visitor)); - } + return visitor.struct(struct, fieldFutures(struct.fields(), visitor)); - return visitor.struct(struct, Iterables.transform(results, VisitFieldFuture::get)); + case FILE: + Types.FileType file = type.asFileType(); + return visitor.file(file, fieldFutures(file.fields(), visitor)); case LIST: Types.ListType list = type.asNestedType().asListType(); @@ -933,6 +1013,15 @@ public static T visit(Type type, CustomOrderSchemaVisitor visitor) { } } + private static Iterable fieldFutures( + List fields, CustomOrderSchemaVisitor visitor) { + List> results = Lists.newArrayListWithExpectedSize(fields.size()); + for (Types.NestedField field : fields) { + results.add(new VisitFieldFuture<>(field, visitor)); + } + return Iterables.transform(results, VisitFieldFuture::get); + } + static int decimalMaxPrecision(int numBytes) { Preconditions.checkArgument( numBytes >= 0 && numBytes < 24, "Unsupported decimal length: %s", numBytes); diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java index f082915920ea..3334fe38c41f 100644 --- a/api/src/main/java/org/apache/iceberg/types/Types.java +++ b/api/src/main/java/org/apache/iceberg/types/Types.java @@ -1155,6 +1155,147 @@ private Map lazyFieldsById() { } } + public static final class FileType extends NestedType { + public static final String NAME = "file"; + public static final int NUM_NESTED_FIELDS = 6; + + private static final String URI = "uri"; + private static final String OFFSET = "offset"; + private static final String SIZE = "size"; + private static final String CONTENT_TYPE = "content_type"; + private static final String CHECKSUM = "checksum"; + private static final String INLINE = "inline"; + + /** + * Returns a file type for the field with the given ID. + * + *

The nested fields of the file use the {@link #NUM_NESTED_FIELDS} IDs that follow {@code + * baseId}. {@code baseId} must be the ID of the field that holds this type. UpdateSchema + * assigns new IDs to each column that it adds. A file type for a new column can use any {@code + * baseId}. + * + * @param baseId the ID of the field that holds this type + * @return a file type with nested field IDs derived from baseId + */ + public static FileType of(int baseId) { + return new FileType(baseId); + } + + private final int baseId; + + // Lazy and derived from baseId + private transient List fieldList = null; + + private FileType(int baseId) { + this.baseId = baseId; + } + + /** Returns the ID of the field that holds this type. */ + public int baseId() { + return baseId; + } + + @Override + public TypeID typeId() { + return TypeID.FILE; + } + + @Override + public boolean isFileType() { + return true; + } + + @Override + public FileType asFileType() { + return this; + } + + @Override + public List fields() { + return lazyFieldList(); + } + + public NestedField field(String name) { + for (NestedField field : fields()) { + if (field.name().equals(name)) { + return field; + } + } + + return null; + } + + @Override + public NestedField field(int id) { + int index = id - baseId - 1; + if (index >= 0 && index < NUM_NESTED_FIELDS) { + return fields().get(index); + } + + return null; + } + + public NestedField caseInsensitiveField(String name) { + for (NestedField field : fields()) { + if (field.name().equalsIgnoreCase(name)) { + return field; + } + } + + return null; + } + + @Override + public Type fieldType(String name) { + NestedField field = field(name); + if (field != null) { + return field.type(); + } + return null; + } + + /** Returns the nested fields of this type as a struct. */ + public StructType asStruct() { + return StructType.of(fields()); + } + + @Override + public String toString() { + return NAME; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } else if (!(other instanceof FileType)) { + return false; + } + + return baseId == ((FileType) other).baseId; + } + + @Override + public int hashCode() { + return Objects.hash(FileType.class, baseId); + } + + private List lazyFieldList() { + if (fieldList == null) { + this.fieldList = + ImmutableList.of( + NestedField.optional(baseId + 1, URI, StringType.get()), + NestedField.optional(baseId + 2, OFFSET, LongType.get()), + NestedField.optional(baseId + 3, SIZE, LongType.get()), + NestedField.optional(baseId + 4, CONTENT_TYPE, StringType.get()), + NestedField.optional(baseId + 5, CHECKSUM, StringType.get()), + NestedField.optional(baseId + 6, INLINE, BinaryType.get())); + } + + return fieldList; + } + } + public static class ListType extends NestedType { public static ListType ofOptional(int elementId, Type elementType) { Preconditions.checkNotNull(elementType, "Element type cannot be null"); diff --git a/api/src/test/java/org/apache/iceberg/TestSchema.java b/api/src/test/java/org/apache/iceberg/TestSchema.java index 7abc3505d52e..e410ee682991 100644 --- a/api/src/test/java/org/apache/iceberg/TestSchema.java +++ b/api/src/test/java/org/apache/iceberg/TestSchema.java @@ -180,6 +180,32 @@ public void testUnknownSupport() { .doesNotThrowAnyException(); } + @Test + void fileSupport() { + // this needs a different schema because a file reserves the six ids that follow it + Schema schemaWithFile = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "top", Types.FileType.of(2)), + Types.NestedField.optional( + 9, "arr", Types.ListType.ofOptional(10, Types.FileType.of(10)))); + int minVersion = MIN_FORMAT_VERSIONS.get(Type.TypeID.FILE); + + for (int version = 1; version < minVersion; version += 1) { + int unsupportedVersion = version; + assertThatThrownBy(() -> Schema.checkCompatibility(schemaWithFile, unsupportedVersion)) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Invalid schema for v%s:\n" + + "- Invalid type for top: file is not supported until v%s\n" + + "- Invalid type for arr.element: file is not supported until v%s", + unsupportedVersion, minVersion, minVersion); + } + + assertThatCode(() -> Schema.checkCompatibility(schemaWithFile, minVersion)) + .doesNotThrowAnyException(); + } + @ParameterizedTest @MethodSource("supportedTypes") public void testTypeSupported(Type type, int supportedVersion) { diff --git a/api/src/test/java/org/apache/iceberg/types/TestFileType.java b/api/src/test/java/org/apache/iceberg/types/TestFileType.java new file mode 100644 index 000000000000..661ac5629070 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/types/TestFileType.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.types; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.iceberg.Schema; +import org.junit.jupiter.api.Test; + +class TestFileType { + private static final Types.FileType FILE = Types.FileType.of(5); + + @Test + void nestedFieldsAreDerivedFromTheBaseId() { + assertThat(FILE.fields()) + .containsExactly( + optional(6, "uri", Types.StringType.get()), + optional(7, "offset", Types.LongType.get()), + optional(8, "size", Types.LongType.get()), + optional(9, "content_type", Types.StringType.get()), + optional(10, "checksum", Types.StringType.get()), + optional(11, "inline", Types.BinaryType.get())); + assertThat(FILE.baseId()).isEqualTo(5); + assertThat(Types.FileType.NUM_NESTED_FIELDS).isEqualTo(FILE.fields().size()); + } + + @Test + void findsNestedFieldsById() { + assertThat(FILE.field(6)).isEqualTo(FILE.fields().get(0)); + assertThat(FILE.field(11)).isEqualTo(FILE.fields().get(5)); + + assertThat(FILE.field(5)).isNull(); + assertThat(FILE.field(12)).isNull(); + } + + @Test + void findsNestedFieldsByName() { + assertThat(FILE.field("content_type")).isEqualTo(FILE.fields().get(3)); + assertThat(FILE.caseInsensitiveField("Content_Type")).isEqualTo(FILE.fields().get(3)); + assertThat(FILE.fieldType("content_type")).isEqualTo(Types.StringType.get()); + + assertThat(FILE.field("missing")).isNull(); + assertThat(FILE.caseInsensitiveField("missing")).isNull(); + assertThat(FILE.fieldType("missing")).isNull(); + } + + @Test + void isDistinguishableFromAStruct() { + assertThat(FILE.isFileType()).isTrue(); + assertThat(FILE.asFileType()).isSameAs(FILE); + + Types.StructType struct = Types.StructType.of(FILE.fields()); + assertThat(struct.isFileType()).isFalse(); + assertThatThrownBy(struct::asFileType) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Not a file type:"); + } + + @Test + void toStringIsFile() { + assertThat(FILE).hasToString("file"); + assertThat(optional(5, "photo", FILE)).hasToString("5: photo: optional file"); + } + + @Test + void isNotEqualToAStructWithTheSameFields() { + Types.StructType struct = Types.StructType.of(FILE.fields()); + + assertThat(FILE).isNotEqualTo(struct); + assertThat(struct).isNotEqualTo(FILE); + } + + @Test + void isEqualOnlyToAFileWithTheSameBaseId() { + assertThat(FILE).isEqualTo(Types.FileType.of(5)).isNotEqualTo(Types.FileType.of(12)); + assertThat(FILE.hashCode()).isEqualTo(Types.FileType.of(5).hashCode()); + } + + @Test + void freshIdsReserveTheNestedIdBlock() { + Schema schema = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101)), + optional(108, "data", Types.StringType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema assigned = TypeUtil.assignFreshIds(schema, new AtomicInteger(0)::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsHandleAdjacentFileColumns() { + Schema schema = + new Schema( + optional(100, "photo", Types.FileType.of(100)), + optional(107, "thumbnail", Types.FileType.of(107)), + optional(114, "data", Types.StringType.get())); + Schema expected = + new Schema( + optional(1, "photo", Types.FileType.of(1)), + optional(8, "thumbnail", Types.FileType.of(8)), + optional(15, "data", Types.StringType.get())); + + Schema assigned = TypeUtil.assignFreshIds(schema, new AtomicInteger(0)::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsReuseBaseSchemaIdsWhenTheBaseColumnIsAlsoAFile() { + Schema base = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + Schema updated = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101)), + optional(108, "data", Types.StringType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema assigned = + TypeUtil.assignFreshIds( + updated, base, new AtomicInteger(base.highestFieldId())::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsReserveANewBlockWhenABaseColumnBecomesAFile() { + Schema base = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.StringType.get()), + optional(3, "data", Types.StringType.get())); + Schema updated = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101)), + optional(108, "data", Types.StringType.get()), + optional(109, "notes", Types.StringType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(4, "photo", Types.FileType.of(4)), + optional(3, "data", Types.StringType.get()), + optional(11, "notes", Types.StringType.get())); + + Schema assigned = + TypeUtil.assignFreshIds( + updated, base, new AtomicInteger(base.highestFieldId())::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsReuseBaseSchemaIdsForAFileInANestedStruct() { + Schema base = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "meta", Types.StructType.of(optional(3, "photo", Types.FileType.of(3))))); + Schema updated = + new Schema( + required(100, "id", Types.LongType.get()), + optional( + 101, + "meta", + Types.StructType.of( + optional(102, "photo", Types.FileType.of(102)), + optional(109, "label", Types.StringType.get())))); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional( + 2, + "meta", + Types.StructType.of( + optional(3, "photo", Types.FileType.of(3)), + optional(10, "label", Types.StringType.get())))); + + Schema assigned = + TypeUtil.assignFreshIds( + updated, base, new AtomicInteger(base.highestFieldId())::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsReserveForFilesInListsAndMaps() { + Schema schema = + new Schema( + optional(100, "photos", Types.ListType.ofOptional(101, Types.FileType.of(101))), + optional( + 108, + "byName", + Types.MapType.ofOptional( + 109, 110, Types.StringType.get(), Types.FileType.of(110)))); + Schema expected = + new Schema( + optional(1, "photos", Types.ListType.ofOptional(3, Types.FileType.of(3))), + optional( + 2, + "byName", + Types.MapType.ofOptional(10, 11, Types.StringType.get(), Types.FileType.of(11)))); + + Schema assigned = TypeUtil.assignFreshIds(schema, new AtomicInteger(0)::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsReuseBaseSchemaIdsForFilesInListsAndMaps() { + Schema base = + new Schema( + optional(1, "photos", Types.ListType.ofOptional(2, Types.FileType.of(2))), + optional( + 9, + "byName", + Types.MapType.ofOptional(10, 11, Types.StringType.get(), Types.FileType.of(11)))); + Schema updated = + new Schema( + optional(100, "photos", Types.ListType.ofOptional(101, Types.FileType.of(101))), + optional( + 108, + "byName", + Types.MapType.ofOptional(109, 110, Types.StringType.get(), Types.FileType.of(110))), + optional(117, "data", Types.StringType.get())); + Schema expected = + new Schema( + optional(1, "photos", Types.ListType.ofOptional(2, Types.FileType.of(2))), + optional( + 9, + "byName", + Types.MapType.ofOptional(10, 11, Types.StringType.get(), Types.FileType.of(11))), + optional(18, "data", Types.StringType.get())); + + Schema assigned = + TypeUtil.assignFreshIds( + updated, base, new AtomicInteger(base.highestFieldId())::incrementAndGet); + + assertThat(assigned.asStruct()).isEqualTo(expected.asStruct()); + } + + @Test + void freshIdsRejectAnAssignerThatSkipsTheReservedIds() { + Schema schema = new Schema(optional(1, "photo", Types.FileType.of(1))); + AtomicInteger counter = new AtomicInteger(0); + + assertThatThrownBy(() -> TypeUtil.assignFreshIds(schema, () -> counter.addAndGet(10))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Failed to assign consecutive IDs 11-16, requested 11 and got 20"); + } + + @Test + void assignedIdsRejectAnAssignerThatCannotReserve() { + Schema schema = new Schema(optional(1, "photo", Types.FileType.of(1))); + + assertThatThrownBy(() -> TypeUtil.assignIds(schema.asStruct(), oldId -> oldId + 10)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessage( + "Cannot assign IDs to file field 1: this GetID does not implement get(int, Type)"); + } + + @Test + void reassignedIdsComeFromTheSourceSchema() { + Schema source = + new Schema( + required(1, "id", Types.LongType.get()), optional(2, "photo", Types.FileType.of(2))); + Schema unassigned = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101))); + + Schema reassigned = TypeUtil.reassignIds(unassigned, source); + + assertThat(reassigned.asStruct()).isEqualTo(source.asStruct()); + } + + @Test + void reassignedIdsComeFromTheSourceSchemaForAFileInANestedStruct() { + Schema source = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "meta", Types.StructType.of(optional(3, "photo", Types.FileType.of(3))))); + Schema unassigned = + new Schema( + required(100, "id", Types.LongType.get()), + optional( + 101, "meta", Types.StructType.of(optional(102, "photo", Types.FileType.of(102))))); + + Schema reassigned = TypeUtil.reassignIds(unassigned, source); + + assertThat(reassigned.asStruct()).isEqualTo(source.asStruct()); + } + + @Test + void refreshedIdsReserveTheNestedIdBlockForNewFileColumns() { + Schema source = new Schema(required(1, "id", Types.LongType.get())); + Schema unassigned = + new Schema( + required(100, "id", Types.LongType.get()), + optional(101, "photo", Types.FileType.of(101)), + optional(108, "data", Types.StringType.get())); + Schema expected = + new Schema( + required(1, "id", Types.LongType.get()), + optional(2, "photo", Types.FileType.of(2)), + optional(9, "data", Types.StringType.get())); + + Schema reassigned = TypeUtil.reassignOrRefreshIds(unassigned, source); + + assertThat(reassigned.asStruct()).isEqualTo(expected.asStruct()); + } +} diff --git a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java index 2363bd8dc66b..8e4fe35c173d 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestSerializableTypes.java @@ -25,6 +25,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.TestHelpers; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; public class TestSerializableTypes { @Test @@ -100,6 +102,19 @@ public void testStructs() throws Exception { .isEqualTo(Types.DecimalType.of(38, 2)); } + @ParameterizedTest + @MethodSource("org.apache.iceberg.TestHelpers#serializers") + public void testFiles(TestHelpers.RoundTripSerializer serializer) throws Exception { + Types.FileType file = Types.FileType.of(5); + + Type copy = serializer.apply(file); + + assertThat(copy).as("File serialization should be equal to starting type").isEqualTo(file); + assertThat(copy.isFileType()).as("File serialization should preserve the file type").isTrue(); + assertThat(copy.asFileType().baseId()).isEqualTo(5); + assertThat(copy.asFileType().fields()).isEqualTo(file.fields()); + } + @Test public void testMaps() throws Exception { Type[] maps = diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index d540d239614e..509f81c2105d 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -793,6 +793,10 @@ private static Stream testTypes() { Arguments.of(Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY))); } + private static Stream testTypesWithFile() { + return Stream.concat(testTypes(), Stream.of(Arguments.of(Types.FileType.of(1)))); + } + @ParameterizedTest @MethodSource("testTypes") public void testAssignIdsWithType(Type testType) { @@ -828,7 +832,7 @@ public void testReassignIdsWithType(Type testType) { } @ParameterizedTest - @MethodSource("testTypes") + @MethodSource("testTypesWithFile") public void testIndexByIdWithType(Type testType) { Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); @@ -837,7 +841,7 @@ public void testIndexByIdWithType(Type testType) { } @ParameterizedTest - @MethodSource("testTypes") + @MethodSource("testTypesWithFile") public void testIndexNameByIdWithType(Type testType) { Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); @@ -864,8 +868,18 @@ public void testGetProjectedIdsWithType(Type testType) { assertThat(Set.of(0, 1)).isEqualTo(projectedIds); } + @Test + void getProjectedIdsSelectsAFileAndItsNestedFields() { + Types.FileType file = Types.FileType.of(1); + Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", file)); + Set expectedIds = Sets.newHashSet(0, 1); + file.fields().forEach(field -> expectedIds.add(field.fieldId())); + + assertThat(TypeUtil.getProjectedIds(schema)).isEqualTo(expectedIds); + } + @ParameterizedTest - @MethodSource("testTypes") + @MethodSource("testTypesWithFile") public void testReassignDocWithType(Type testType) { Schema schema = new Schema(required(0, "id", IntegerType.get()), optional(1, "data", testType)); Schema docSourceSchema = diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypes.java b/api/src/test/java/org/apache/iceberg/types/TestTypes.java index 2fb224aefb15..cf2ca80190e7 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypes.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypes.java @@ -62,6 +62,11 @@ public void fromTypeName() { assertThat(Types.fromTypeName("geography ( srid:4269 , karney )")) .isEqualTo(Types.GeographyType.of("srid:4269", EdgeAlgorithm.KARNEY)); + // a file is not resolvable by name because its nested ids come from the field that holds it + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> Types.fromTypeName("file")) + .withMessage("Cannot parse type string to primitive: file"); + assertThatExceptionOfType(IllegalArgumentException.class) .isThrownBy(() -> Types.fromTypeName("abcdefghij")) .withMessage("Cannot parse type string to primitive: abcdefghij"); @@ -92,6 +97,10 @@ public void fromPrimitiveString() { .isThrownBy(() -> Types.fromPrimitiveString("Variant")) .withMessage("Cannot parse type string: variant is not a primitive type"); + assertThatExceptionOfType(IllegalArgumentException.class) + .isThrownBy(() -> Types.fromPrimitiveString("file")) + .withMessage("Cannot parse type string to primitive: file"); + assertThatExceptionOfType(IllegalArgumentException.class) .isThrownBy(() -> Types.fromPrimitiveString("abcdefghij")) .withMessage("Cannot parse type string to primitive: abcdefghij");