diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 81d5fa6072a0..eaabffc2a2db 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -1040,6 +1040,15 @@ acceptedBreaks: justification: "Removing deprecated createStructReader overload scheduled for\ \ removal in 1.12.0" "1.12.0": + org.apache.iceberg:iceberg-api: + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.PartitionField" + new: "class org.apache.iceberg.PartitionField" + justification: "Adding source-ids field to support multi-arg transforms" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.SortField" + new: "class org.apache.iceberg.SortField" + justification: "Adding source-ids field to support multi-arg transforms" org.apache.iceberg:iceberg-core: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.rest.RESTCatalogProperties.SIGNER_ENDPOINT" diff --git a/api/src/main/java/org/apache/iceberg/PartitionField.java b/api/src/main/java/org/apache/iceberg/PartitionField.java index 3ed765a89834..119cc86586ac 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionField.java +++ b/api/src/main/java/org/apache/iceberg/PartitionField.java @@ -19,18 +19,28 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.base.Objects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; /** Represents a single field in a {@link PartitionSpec}. */ public class PartitionField implements Serializable { - private final int sourceId; + private static final Joiner COMMA = Joiner.on(", "); + + private final List sourceIds; private final int fieldId; private final String name; private final Transform transform; PartitionField(int sourceId, int fieldId, String name, Transform transform) { - this.sourceId = sourceId; + this(ImmutableList.of(sourceId), fieldId, name, transform); + } + + PartitionField(List sourceIds, int fieldId, String name, Transform transform) { + this.sourceIds = ImmutableList.copyOf(sourceIds); this.fieldId = fieldId; this.name = name; this.transform = transform; @@ -38,7 +48,21 @@ public class PartitionField implements Serializable { /** Returns the field id of the source field in the {@link PartitionSpec spec's} table schema. */ public int sourceId() { - return sourceId; + Preconditions.checkState( + sourceIds.size() == 1, + "Use sourceIds() for multi-argument transforms, found %s source ids", + sourceIds.size()); + return sourceIds.get(0); + } + + /** + * Returns the field ids of all source fields for this partition field. + * + *

For single-argument transforms, this list contains one element. For multi-argument + * transforms, this list contains multiple source field ids. + */ + public List sourceIds() { + return sourceIds; } /** Returns the partition field id across all the table metadata's partition specs. */ @@ -58,7 +82,7 @@ public String name() { @Override public String toString() { - return fieldId + ": " + name + ": " + transform + "(" + sourceId + ")"; + return fieldId + ": " + name + ": " + transform + "(" + COMMA.join(sourceIds) + ")"; } @Override @@ -70,7 +94,7 @@ public boolean equals(Object other) { } PartitionField that = (PartitionField) other; - return sourceId == that.sourceId + return sourceIds.equals(that.sourceIds) && fieldId == that.fieldId && name.equals(that.name) && transform.toString().equals(that.transform.toString()); @@ -78,6 +102,6 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hashCode(sourceId, fieldId, name, transform); + return Objects.hashCode(sourceIds, fieldId, name, transform); } } diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index 9c1528582c8a..cb1e01862fe8 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -115,7 +115,7 @@ public UnboundPartitionSpec toUnbound() { for (PartitionField field : fields) { builder.addField( - field.transform().toString(), field.sourceId(), field.fieldId(), field.name()); + field.transform().toString(), field.sourceIds(), field.fieldId(), field.name()); } return builder.build(); @@ -198,7 +198,7 @@ public Class[] javaClasses() { } private Type resultType(PartitionField field) { - Type sourceType = schema.findType(field.sourceId()); + Type sourceType = schema.findType(field.sourceIds().get(0)); if (sourceType == null) { // When the source field has been dropped, the source type has been lost // Transforms with a fixed result type still work, so use unknown for source @@ -254,7 +254,7 @@ public boolean compatibleWith(PartitionSpec other) { for (int i = 0; i < fields.length; i += 1) { PartitionField thisField = fields[i]; PartitionField thatField = other.fields[i]; - if (thisField.sourceId() != thatField.sourceId() + if (!thisField.sourceIds().equals(thatField.sourceIds()) || !thisField.transform().toString().equals(thatField.transform().toString()) || !thisField.name().equals(thatField.name())) { return false; @@ -303,7 +303,9 @@ private ListMultimap lazyFieldsBySourceId() { Multimaps.newListMultimap( Maps.newHashMap(), () -> Lists.newArrayListWithCapacity(fields.length)); for (PartitionField field : fields) { - multiMap.put(field.sourceId(), field); + for (int sourceId : field.sourceIds()) { + multiMap.put(sourceId, field); + } } this.fieldsBySourceId = multiMap; } @@ -322,7 +324,7 @@ public Set identitySourceIds() { Set sourceIds = Sets.newHashSet(); for (PartitionField field : fields()) { if (field.transform().isIdentity()) { - sourceIds.add(field.sourceId()); + sourceIds.addAll(field.sourceIds()); } } @@ -433,15 +435,17 @@ private void checkAndAddPartitionName(String name, Integer sourceColumnId) { } private void checkForRedundantPartitions(PartitionField field) { - Map.Entry dedupKey = - new AbstractMap.SimpleEntry<>(field.sourceId(), field.transform().dedupName()); - PartitionField partitionField = dedupFields.get(dedupKey); - Preconditions.checkArgument( - partitionField == null, - "Cannot add redundant partition: %s conflicts with %s", - partitionField, - field); - dedupFields.put(dedupKey, field); + for (int sourceId : field.sourceIds()) { + Map.Entry dedupKey = + new AbstractMap.SimpleEntry<>(sourceId, field.transform().dedupName()); + PartitionField partitionField = dedupFields.get(dedupKey); + Preconditions.checkArgument( + partitionField == null, + "Cannot add redundant partition: %s conflicts with %s", + partitionField, + field); + dedupFields.put(dedupKey, field); + } } public Builder caseSensitive(boolean sensitive) { @@ -627,6 +631,22 @@ Builder add(int sourceId, int fieldId, String name, Transform transform) { return this; } + Builder add(List sourceIds, int fieldId, String name, Transform transform) { + if (sourceIds.size() == 1) { + checkAndAddPartitionName(name, sourceIds.get(0)); + } else { + checkAndAddPartitionName(name); + } + + fields.add(new PartitionField(sourceIds, fieldId, name, transform)); + lastAssignedFieldId.getAndAccumulate(fieldId, Math::max); + return this; + } + + Builder add(List sourceIds, String name, Transform transform) { + return add(sourceIds, nextFieldId(), name, transform); + } + public PartitionSpec build() { return build(false); } @@ -649,37 +669,39 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) { static void checkCompatibility(PartitionSpec spec, Schema schema, boolean allowMissingFields) { final Map parents = TypeUtil.indexParents(schema.asStruct()); for (PartitionField field : spec.fields) { - Type sourceType = schema.findType(field.sourceId()); Transform transform = field.transform(); - // In the case the underlying field is dropped, we cannot check if they are compatible - if (allowMissingFields && sourceType == null) { - continue; - } - // In the case of a Version 1 partition-spec field gets deleted, - // it is replaced with a void transform, see: - // https://iceberg.apache.org/spec/#partition-transforms - // We don't care about the source type since a VoidTransform is always compatible and skip the - // checks - if (!transform.equals(Transforms.alwaysNull())) { - ValidationException.check( - sourceType != null, "Cannot find source column for partition field: %s", field); - ValidationException.check( - sourceType.isPrimitiveType(), - "Cannot partition by non-primitive source field: %s", - sourceType); - ValidationException.check( - transform.canTransform(sourceType), - "Invalid source type %s for transform: %s", - sourceType, - transform); - // The only valid parent types for a PartitionField are StructTypes. This must be checked - // recursively. - Integer parentId = parents.get(field.sourceId()); - while (parentId != null) { - Type parentType = schema.findType(parentId); + for (int srcId : field.sourceIds()) { + Type sourceType = schema.findType(srcId); + // In the case the underlying field is dropped, we cannot check if they are compatible + if (allowMissingFields && sourceType == null) { + continue; + } + // In the case of a Version 1 partition-spec field gets deleted, + // it is replaced with a void transform, see: + // https://iceberg.apache.org/spec/#partition-transforms + // We don't care about the source type since a VoidTransform is always compatible and skip + // the checks + if (!transform.equals(Transforms.alwaysNull())) { + ValidationException.check( + sourceType != null, "Cannot find source column for partition field: %s", field); ValidationException.check( - parentType.isStructType(), "Invalid partition field parent: %s", parentType); - parentId = parents.get(parentId); + sourceType.isPrimitiveType(), + "Cannot partition by non-primitive source field: %s", + sourceType); + ValidationException.check( + transform.canTransform(sourceType), + "Invalid source type %s for transform: %s", + sourceType, + transform); + // The only valid parent types for a PartitionField are StructTypes. This must be checked + // recursively. + Integer parentId = parents.get(srcId); + while (parentId != null) { + Type parentType = schema.findType(parentId); + ValidationException.check( + parentType.isStructType(), "Invalid partition field parent: %s", parentType); + parentId = parents.get(parentId); + } } } } diff --git a/api/src/main/java/org/apache/iceberg/SortField.java b/api/src/main/java/org/apache/iceberg/SortField.java index d7f110a26e3f..b3dab01abc81 100644 --- a/api/src/main/java/org/apache/iceberg/SortField.java +++ b/api/src/main/java/org/apache/iceberg/SortField.java @@ -19,20 +19,31 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.List; import java.util.Objects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; /** A field in a {@link SortOrder}. */ public class SortField implements Serializable { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final SortDirection direction; private final NullOrder nullOrder; SortField(Transform transform, int sourceId, SortDirection direction, NullOrder nullOrder) { + this(transform, ImmutableList.of(sourceId), direction, nullOrder); + } + + SortField( + Transform transform, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { this.transform = transform; - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.direction = direction; this.nullOrder = nullOrder; } @@ -51,7 +62,21 @@ public Transform transform() { /** Returns the field id of the source field in the {@link SortOrder sort order's} table schema */ public int sourceId() { - return sourceId; + Preconditions.checkState( + sourceIds.size() == 1, + "Use sourceIds() for multi-argument transforms, found %s source ids", + sourceIds.size()); + return sourceIds.get(0); + } + + /** + * Returns the field ids of all source fields for this sort field. + * + *

For single-argument transforms, this list contains one element. For multi-argument + * transforms, this list contains multiple source field ids. + */ + public List sourceIds() { + return sourceIds; } /** Returns the sort direction */ @@ -73,7 +98,7 @@ public NullOrder nullOrder() { public boolean satisfies(SortField other) { if (Objects.equals(this, other)) { return true; - } else if (sourceId != other.sourceId + } else if (!sourceIds.equals(other.sourceIds) || direction != other.direction || nullOrder != other.nullOrder) { return false; @@ -84,7 +109,10 @@ public boolean satisfies(SortField other) { @Override public String toString() { - return transform + "(" + sourceId + ") " + direction + " " + nullOrder; + if (sourceIds.size() == 1) { + return transform + "(" + sourceIds.get(0) + ") " + direction + " " + nullOrder; + } + return transform + "(" + sourceIds + ") " + direction + " " + nullOrder; } @Override @@ -97,13 +125,13 @@ public boolean equals(Object other) { SortField that = (SortField) other; return transform.toString().equals(that.transform.toString()) - && sourceId == that.sourceId + && sourceIds.equals(that.sourceIds) && direction == that.direction && nullOrder == that.nullOrder; } @Override public int hashCode() { - return Objects.hash(transform, sourceId, direction, nullOrder); + return Objects.hash(transform, sourceIds, direction, nullOrder); } } diff --git a/api/src/main/java/org/apache/iceberg/SortOrder.java b/api/src/main/java/org/apache/iceberg/SortOrder.java index e98e05bd1e88..52127a26856d 100644 --- a/api/src/main/java/org/apache/iceberg/SortOrder.java +++ b/api/src/main/java/org/apache/iceberg/SortOrder.java @@ -126,8 +126,13 @@ public UnboundSortOrder toUnbound() { UnboundSortOrder.Builder builder = UnboundSortOrder.builder().withOrderId(orderId); for (SortField field : fields) { - builder.addSortField( - field.transform().toString(), field.sourceId(), field.direction(), field.nullOrder()); + if (field.sourceIds().size() > 1) { + builder.addSortField( + field.transform().toString(), field.sourceIds(), field.direction(), field.nullOrder()); + } else { + builder.addSortField( + field.transform().toString(), field.sourceId(), field.direction(), field.nullOrder()); + } } return builder.build(); @@ -260,6 +265,16 @@ Builder addSortField( return this; } + Builder addSortField( + Transform transform, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { + SortField sortField = new SortField(transform, sourceIds, direction, nullOrder); + fields.add(sortField); + return this; + } + public SortOrder build() { SortOrder sortOrder = buildUnchecked(); checkCompatibility(sortOrder, schema); @@ -297,18 +312,20 @@ SortOrder buildUnchecked() { public static void checkCompatibility(SortOrder sortOrder, Schema schema) { for (SortField field : sortOrder.fields) { - Type sourceType = schema.findType(field.sourceId()); - ValidationException.check( - sourceType != null, "Cannot find source column for sort field: %s", field); - ValidationException.check( - sourceType.isPrimitiveType(), - "Cannot sort by non-primitive source field: %s", - sourceType); - ValidationException.check( - field.transform().canTransform(sourceType), - "Invalid source type %s for transform: %s", - sourceType, - field.transform()); + for (int srcId : field.sourceIds()) { + Type sourceType = schema.findType(srcId); + ValidationException.check( + sourceType != null, "Cannot find source column for sort field: %s", field); + ValidationException.check( + sourceType.isPrimitiveType(), + "Cannot sort by non-primitive source field: %s", + sourceType); + ValidationException.check( + field.transform().canTransform(sourceType), + "Invalid source type %s for transform: %s", + sourceType, + field.transform()); + } } } } diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java index 30b3cce35fc8..d359c46439f8 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; @@ -58,17 +59,24 @@ private PartitionSpec.Builder copyToBuilder(Schema schema) { PartitionSpec.Builder builder = PartitionSpec.builderFor(schema).withSpecId(specId); for (UnboundPartitionField field : fields) { - Type fieldType = schema.findType(field.sourceId); - Transform transform; - if (fieldType != null) { - transform = Transforms.fromString(fieldType, field.transform.toString()); - } else { - transform = Transforms.fromString(field.transform.toString()); + List sourceTypes = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + sourceTypes.add(schema.findType(sourceId)); } - if (field.partitionId != null) { - builder.add(field.sourceId, field.partitionId, field.name, transform); + + Transform transform = Transforms.fromString(sourceTypes, field.transformAsString()); + if (field.sourceIds().size() > 1) { + if (field.partitionId() != null) { + builder.add(field.sourceIds(), field.partitionId(), field.name(), transform); + } else { + builder.add(field.sourceIds(), field.name(), transform); + } } else { - builder.add(field.sourceId, field.name, transform); + if (field.partitionId() != null) { + builder.add(field.sourceId(), field.partitionId(), field.name(), transform); + } else { + builder.add(field.sourceId(), field.name(), transform); + } } } @@ -102,6 +110,17 @@ Builder addField(String transformAsString, int sourceId, String name) { return this; } + Builder addField( + String transformAsString, List sourceIds, int partitionId, String name) { + fields.add(new UnboundPartitionField(transformAsString, sourceIds, partitionId, name)); + return this; + } + + Builder addField(String transformAsString, List sourceIds, String name) { + fields.add(new UnboundPartitionField(transformAsString, sourceIds, null, name)); + return this; + } + UnboundPartitionSpec build() { return new UnboundPartitionSpec(specId, fields); } @@ -109,7 +128,7 @@ UnboundPartitionSpec build() { static class UnboundPartitionField { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final Integer partitionId; private final String name; @@ -122,7 +141,11 @@ public String transformAsString() { } public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + public List sourceIds() { + return sourceIds; } public Integer partitionId() { @@ -135,8 +158,13 @@ public String name() { private UnboundPartitionField( String transformAsString, int sourceId, Integer partitionId, String name) { + this(transformAsString, ImmutableList.of(sourceId), partitionId, name); + } + + private UnboundPartitionField( + String transformAsString, List sourceIds, Integer partitionId, String name) { this.transform = Transforms.fromString(transformAsString); - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.partitionId = partitionId; this.name = name; } diff --git a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java index ce9f6b1d6b2c..f815c438d5c1 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java +++ b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; @@ -41,14 +42,17 @@ public SortOrder bind(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - Type sourceType = schema.findType(field.sourceId); - Transform transform; - if (sourceType != null) { - transform = Transforms.fromString(sourceType, field.transform.toString()); + List sourceTypes = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + sourceTypes.add(schema.findType(sourceId)); + } + + Transform transform = Transforms.fromString(sourceTypes, field.transformAsString()); + if (field.sourceIds().size() > 1) { + builder.addSortField(transform, field.sourceIds(), field.direction, field.nullOrder); } else { - transform = field.transform; + builder.addSortField(transform, field.sourceId(), field.direction, field.nullOrder); } - builder.addSortField(transform, field.sourceId, field.direction, field.nullOrder); } return builder.build(); @@ -58,7 +62,11 @@ SortOrder bindUnchecked(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder); + if (field.sourceIds().size() > 1) { + builder.addSortField(field.transform, field.sourceIds(), field.direction, field.nullOrder); + } else { + builder.addSortField(field.transform, field.sourceId(), field.direction, field.nullOrder); + } } return builder.buildUnchecked(); @@ -103,6 +111,15 @@ Builder addSortField( return this; } + Builder addSortField( + String transformAsString, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { + fields.add(new UnboundSortField(transformAsString, sourceIds, direction, nullOrder)); + return this; + } + UnboundSortOrder build() { if (fields.isEmpty()) { if (orderId != null && orderId != 0) { @@ -123,14 +140,22 @@ UnboundSortOrder build() { static class UnboundSortField { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final SortDirection direction; private final NullOrder nullOrder; private UnboundSortField( String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) { + this(transformAsString, ImmutableList.of(sourceId), direction, nullOrder); + } + + private UnboundSortField( + String transformAsString, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { this.transform = Transforms.fromString(transformAsString); - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.direction = direction; this.nullOrder = nullOrder; } @@ -140,7 +165,11 @@ public String transformAsString() { } public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + public List sourceIds() { + return sourceIds; } public SortDirection direction() { diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java index 958abd27375a..ed02cd0ea584 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java @@ -18,11 +18,13 @@ */ package org.apache.iceberg.transforms; +import java.util.List; import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; /** @@ -115,6 +117,29 @@ private Transforms() {} return new UnknownTransform<>(transform); } + /** + * Parses a transform from its string representation for the given source types. + * + *

Multi-argument transforms are not recognized and produce an {@link UnknownTransform}. A + * {@code null} source type, for a source field that no longer exists, produces the same result as + * {@link #fromString(String)}. + * + * @param types the source types of the transform arguments, in order + * @param transform a transform string, such as {@code "day"} or {@code "bucket[16]"} + * @return the parsed transform + */ + public static Transform fromString(List types, String transform) { + Preconditions.checkArgument(types != null, "Invalid source types: null"); + Preconditions.checkArgument(!types.isEmpty(), "Invalid source types: empty"); + + if (types.size() > 1) { + return new UnknownTransform<>(transform); + } + + Type type = types.get(0); + return type != null ? fromString(type, transform) : fromString(transform); + } + /** * Returns an identity {@link Transform} that can be used for any type. * diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index 5c37ad527291..0185f1ca6328 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; @@ -242,6 +243,16 @@ public void testSettingPartitionTransformsWithCustomTargetNamesThatAlreadyExist( "Cannot create identity partition sourced from different field in schema: another_ts"); } + @Test + void multiArgPartitionNameMatchingSourceColumn() { + assertThatThrownBy( + () -> + PartitionSpec.builderFor(SCHEMA) + .add(ImmutableList.of(1, 6), "id", Transforms.bucket(8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot create partition from name that exists in schema: id"); + } + @Test public void testStalePartitionSourceIdWithReusedColumnName() { int newFieldId = 2; diff --git a/api/src/test/java/org/apache/iceberg/TestSortField.java b/api/src/test/java/org/apache/iceberg/TestSortField.java new file mode 100644 index 000000000000..27dc59fc4261 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/TestSortField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestSortField { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.IntegerType.get()), required(2, "data", Types.StringType.get())); + + @Test + void sourceIdForSingleArgField() { + SortField field = SortOrder.builderFor(SCHEMA).asc("id").build().fields().get(0); + + assertThat(field.sourceIds()).containsExactly(1); + assertThat(field.sourceId()).isEqualTo(1); + } + + @Test + void sourceIdFailsForMultiArgField() { + SortField field = + UnboundSortOrder.builder() + .withOrderId(1) + .addSortField( + "bucket[16]", ImmutableList.of(1, 2), SortDirection.ASC, NullOrder.NULLS_FIRST) + .build() + .bind(SCHEMA) + .fields() + .get(0); + + assertThat(field.sourceIds()).containsExactly(1, 2); + assertThatThrownBy(field::sourceId) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Use sourceIds() for multi-argument transforms, found 2 source ids"); + } +} diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index c69f6f3844f9..bcb97f74a222 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -30,6 +30,7 @@ import org.apache.iceberg.expressions.UnboundTerm; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -122,7 +123,7 @@ private int assignFieldId() { private PartitionField recycleOrCreatePartitionField( Pair> sourceTransform, String name) { if (formatVersion >= 2 && base != null) { - int sourceId = sourceTransform.first(); + List sourceIds = ImmutableList.of(sourceTransform.first()); Transform transform = sourceTransform.second(); Set allHistoricalFields = Sets.newHashSet(); @@ -131,7 +132,7 @@ private PartitionField recycleOrCreatePartitionField( } for (PartitionField field : allHistoricalFields) { - if (field.sourceId() == sourceId && field.transform().equals(transform)) { + if (field.sourceIds().equals(sourceIds) && field.transform().equals(transform)) { // if target name is specified then consider it too, otherwise not if (name == null || field.name().equals(name)) { return field; @@ -308,9 +309,9 @@ public PartitionSpec apply() { if (!deletes.contains(field.fieldId())) { String newName = renames.get(field.name()); if (newName != null) { - builder.add(field.sourceId(), field.fieldId(), newName, field.transform()); + builder.add(field.sourceIds(), field.fieldId(), newName, field.transform()); } else { - builder.add(field.sourceId(), field.fieldId(), field.name(), field.transform()); + builder.add(field.sourceIds(), field.fieldId(), field.name(), field.transform()); } } else if (formatVersion < 2) { // field IDs were not required for v1 and were assigned sequentially in each partition spec @@ -320,9 +321,9 @@ public PartitionSpec apply() { // must be replaced with a null transform. null values are always allowed in partition data. String newName = renames.get(field.name()); if (newName != null) { - builder.add(field.sourceId(), field.fieldId(), newName, Transforms.alwaysNull()); + builder.add(field.sourceIds(), field.fieldId(), newName, Transforms.alwaysNull()); } else { - builder.add(field.sourceId(), field.fieldId(), field.name(), Transforms.alwaysNull()); + builder.add(field.sourceIds(), field.fieldId(), field.name(), Transforms.alwaysNull()); } } } @@ -399,7 +400,10 @@ private static Map, PartitionField> indexSpecByTransform( Map, PartitionField> indexSpecs = Maps.newHashMap(); List fields = spec.fields(); for (PartitionField field : fields) { - indexSpecs.put(Pair.of(field.sourceId(), field.transform().toString()), field); + // multi-arg fields cannot be referenced by a term, which always has a single source field + if (field.sourceIds().size() == 1) { + indexSpecs.put(Pair.of(field.sourceId(), field.transform().toString()), field); + } } return indexSpecs; diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index 7becf0c62943..39202758db2d 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -24,17 +24,24 @@ import com.github.benmanes.caffeine.cache.Caffeine; import java.io.IOException; import java.util.Iterator; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PartitionSpecParser { + private static final Logger LOG = LoggerFactory.getLogger(PartitionSpecParser.class); + private PartitionSpecParser() {} private static final String SPEC_ID = "spec-id"; private static final String FIELDS = "fields"; private static final String SOURCE_ID = "source-id"; + private static final String SOURCE_IDS = "source-ids"; private static final String FIELD_ID = "field-id"; private static final String TRANSFORM = "transform"; private static final String NAME = "name"; @@ -98,7 +105,16 @@ static void toJsonFields(UnboundPartitionSpec spec, JsonGenerator generator) thr generator.writeStartObject(); generator.writeStringField(NAME, field.name()); generator.writeStringField(TRANSFORM, field.transformAsString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + if (field.sourceIds().size() == 1) { + generator.writeNumberField(SOURCE_ID, field.sourceId()); + } else { + generator.writeFieldName(SOURCE_IDS); + generator.writeStartArray(); + for (Integer value : field.sourceIds()) { + generator.writeNumber(value); + } + generator.writeEndArray(); + } generator.writeNumberField(FIELD_ID, field.partitionId()); generator.writeEndObject(); } @@ -132,15 +148,48 @@ private static void buildFromJsonFields(UnboundPartitionSpec.Builder builder, Js String name = JsonUtil.getString(NAME, element); String transform = JsonUtil.getString(TRANSFORM, element); - int sourceId = JsonUtil.getInt(SOURCE_ID, element); - // partition field ids are missing in old PartitionSpec, they always auto-increment from - // PARTITION_DATA_ID_START - if (element.has(FIELD_ID)) { - builder.addField(transform, sourceId, JsonUtil.getInt(FIELD_ID, element), name); - fieldIdCount++; + if (element.has(SOURCE_IDS)) { + JsonNode sourceIdsNode = element.get(SOURCE_IDS); + Preconditions.checkArgument( + sourceIdsNode.isArray() && sourceIdsNode.size() > 0, + "Cannot parse partition field, source-ids must be a non-empty array: %s", + element); + List sourceIds = Lists.newArrayListWithCapacity(sourceIdsNode.size()); + for (int i = 0; i < sourceIdsNode.size(); i++) { + sourceIds.add(sourceIdsNode.get(i).asInt()); + } + + if (element.has(SOURCE_ID)) { + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + if (!sourceIds.get(0).equals(sourceId)) { + LOG.warn( + "Conflicting source-id ({}) and source-ids ({}) in partition field: {}", + sourceId, + sourceIds, + element); + } + } + + // partition field ids are missing in old PartitionSpec, they always auto-increment from + // PARTITION_DATA_ID_START + if (element.has(FIELD_ID)) { + builder.addField(transform, sourceIds, JsonUtil.getInt(FIELD_ID, element), name); + fieldIdCount++; + } else { + builder.addField(transform, sourceIds, name); + } } else { - builder.addField(transform, sourceId, name); + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + + // partition field ids are missing in old PartitionSpec, they always auto-increment from + // PARTITION_DATA_ID_START + if (element.has(FIELD_ID)) { + builder.addField(transform, sourceId, JsonUtil.getInt(FIELD_ID, element), name); + fieldIdCount++; + } else { + builder.addField(transform, sourceId, name); + } } } diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index 66a681f88486..b0df5b53aadb 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -348,7 +348,7 @@ private static boolean isVoidTransform(PartitionField field) { private static boolean equivalentIgnoringNames( PartitionField field, PartitionField anotherField) { return field.fieldId() == anotherField.fieldId() - && field.sourceId() == anotherField.sourceId() + && field.sourceIds().equals(anotherField.sourceIds()) && compatibleTransforms(field.transform(), anotherField.transform()); } @@ -370,7 +370,7 @@ private static Set allFieldIds(Collection specs) { private static Set allActiveFieldIds(Schema schema, Collection specs) { return FluentIterable.from(specs) .transformAndConcat(PartitionSpec::fields) - .filter(field -> schema.findField(field.sourceId()) != null) + .filter(field -> hasAllSourceFields(schema, field)) .transform(PartitionField::fieldId) .toSet(); } @@ -395,9 +395,19 @@ private static Set commonActiveFieldIds(Schema schema, Collection activeFieldIds(Schema schema, PartitionSpec spec) { return spec.fields().stream() - .filter(field -> schema == null || schema.findField(field.sourceId()) != null) + .filter(field -> schema == null || hasAllSourceFields(schema, field)) .filter(field -> !isVoidTransform(field)) .map(PartitionField::fieldId) .collect(Collectors.toList()); } + + private static boolean hasAllSourceFields(Schema schema, PartitionField field) { + for (int sourceId : field.sourceIds()) { + if (schema.findField(sourceId) == null) { + return false; + } + } + + return true; + } } diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 74126162aaf2..8edc75cfcdab 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -25,8 +25,10 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Locale; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.util.JsonUtil; public class SortOrderParser { @@ -36,6 +38,7 @@ public class SortOrderParser { private static final String NULL_ORDER = "null-order"; private static final String TRANSFORM = "transform"; private static final String SOURCE_ID = "source-id"; + private static final String SOURCE_IDS = "source-ids"; private SortOrderParser() {} @@ -69,7 +72,7 @@ private static void toJsonFields(SortOrder sortOrder, JsonGenerator generator) for (SortField field : sortOrder.fields()) { generator.writeStartObject(); generator.writeStringField(TRANSFORM, field.transform().toString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + writeSourceIds(field.sourceIds(), generator); generator.writeStringField(DIRECTION, toJson(field.direction())); generator.writeStringField(NULL_ORDER, toJson(field.nullOrder())); generator.writeEndObject(); @@ -100,7 +103,7 @@ private static void toJsonFields(UnboundSortOrder sortOrder, JsonGenerator gener for (UnboundSortOrder.UnboundSortField field : sortOrder.fields()) { generator.writeStartObject(); generator.writeStringField(TRANSFORM, field.transformAsString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + writeSourceIds(field.sourceIds(), generator); generator.writeStringField(DIRECTION, toJson(field.direction())); generator.writeStringField(NULL_ORDER, toJson(field.nullOrder())); generator.writeEndObject(); @@ -108,6 +111,20 @@ private static void toJsonFields(UnboundSortOrder sortOrder, JsonGenerator gener generator.writeEndArray(); } + private static void writeSourceIds(List sourceIds, JsonGenerator generator) + throws IOException { + if (sourceIds.size() == 1) { + generator.writeNumberField(SOURCE_ID, sourceIds.get(0)); + } else { + generator.writeFieldName(SOURCE_IDS); + generator.writeStartArray(); + for (Integer value : sourceIds) { + generator.writeNumber(value); + } + generator.writeEndArray(); + } + } + public static SortOrder fromJson(Schema schema, String json) { return fromJson(json).bind(schema); } @@ -155,7 +172,6 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo element.isObject(), "Cannot parse sort field, not an object: %s", element); String transform = JsonUtil.getString(TRANSFORM, element); - int sourceId = JsonUtil.getInt(SOURCE_ID, element); String directionAsString = JsonUtil.getString(DIRECTION, element); SortDirection direction = SortDirection.fromString(directionAsString); @@ -163,7 +179,21 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo String nullOrderingAsString = JsonUtil.getString(NULL_ORDER, element); NullOrder nullOrder = toNullOrder(nullOrderingAsString); - builder.addSortField(transform, sourceId, direction, nullOrder); + if (element.has(SOURCE_IDS)) { + JsonNode sourceIdsNode = element.get(SOURCE_IDS); + Preconditions.checkArgument( + sourceIdsNode.isArray() && sourceIdsNode.size() > 0, + "Cannot parse sort field, source-ids must be a non-empty array: %s", + element); + List sourceIds = Lists.newArrayListWithCapacity(sourceIdsNode.size()); + for (int i = 0; i < sourceIdsNode.size(); i++) { + sourceIds.add(sourceIdsNode.get(i).asInt()); + } + builder.addSortField(transform, sourceIds, direction, nullOrder); + } else { + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + builder.addSortField(transform, sourceId, direction, nullOrder); + } } } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 89427e4c6943..35f36c81bc57 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -127,10 +127,9 @@ static TableMetadata newTableMetadata( PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(freshSchema).withSpecId(INITIAL_SPEC_ID); for (PartitionField field : spec.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = schema.findColumnName(field.sourceId()); // reassign all partition fields with fresh partition field Ids to ensure consistency - specBuilder.add(freshSchema.findField(sourceName).fieldId(), field.name(), field.transform()); + specBuilder.add( + freshSourceIds(schema, freshSchema, field.sourceIds()), field.name(), field.transform()); } PartitionSpec freshSpec = specBuilder.build(); @@ -652,12 +651,12 @@ private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil if (formatVersion > 1) { // for v2 and later, reuse any existing field IDs, but reproduce the same spec - Map, Integer> transformToFieldId = + Map, String>, Integer> transformToFieldId = specs.stream() .flatMap(spec -> spec.fields().stream()) .collect( Collectors.toMap( - field -> Pair.of(field.sourceId(), field.transform().toString()), + field -> Pair.of(field.sourceIds(), field.transform().toString()), PartitionField::fieldId, Math::max)); @@ -665,8 +664,8 @@ private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil // reassign the partition field ids int partitionFieldId = transformToFieldId.computeIfAbsent( - Pair.of(field.sourceId(), field.transform().toString()), k -> nextID.get()); - specBuilder.add(field.sourceId(), partitionFieldId, field.name(), field.transform()); + Pair.of(field.sourceIds(), field.transform().toString()), k -> nextID.get()); + specBuilder.add(field.sourceIds(), partitionFieldId, field.name(), field.transform()); } } else { @@ -763,7 +762,7 @@ private static PartitionSpec updateSpecSchema(Schema schema, PartitionSpec parti // add all the fields to the builder. IDs should not change. for (PartitionField field : partitionSpec.fields()) { - specBuilder.add(field.sourceId(), field.fieldId(), field.name(), field.transform()); + specBuilder.add(field.sourceIds(), field.fieldId(), field.name(), field.transform()); } // build without validation because the schema may have changed in a way that makes this spec @@ -778,7 +777,7 @@ private static SortOrder updateSortOrderSchema(Schema schema, SortOrder sortOrde // add all the fields to the builder. IDs should not change. for (SortField field : sortOrder.fields()) { builder.addSortField( - field.transform(), field.sourceId(), field.direction(), field.nullOrder()); + field.transform(), field.sourceIds(), field.direction(), field.nullOrder()); } // build without validation because the schema may have changed in a way that makes this order @@ -791,18 +790,20 @@ private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec UnboundPartitionSpec.Builder specBuilder = UnboundPartitionSpec.builder().withSpecId(specId); for (PartitionField field : partitionSpec.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = partitionSpec.schema().findColumnName(field.sourceId()); - - final int fieldId; - if (sourceName != null) { - fieldId = schema.findField(sourceName).fieldId(); - } else { - // In the case of a null sourceName, the column has been deleted. - // This only happens in V1 tables where the reference is still around as a void transform - fieldId = field.sourceId(); + List sourceIds = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + // look up the name of the source field in the old schema to get the new schema's id + String sourceName = partitionSpec.schema().findColumnName(sourceId); + if (sourceName != null) { + sourceIds.add(schema.findField(sourceName).fieldId()); + } else { + // In the case of a null sourceName, the column has been deleted. + // This only happens in V1 tables where the reference is still around as a void transform + sourceIds.add(sourceId); + } } - specBuilder.addField(field.transform().toString(), fieldId, field.fieldId(), field.name()); + + specBuilder.addField(field.transform().toString(), sourceIds, field.fieldId(), field.name()); } return specBuilder.build().bind(schema); @@ -816,17 +817,27 @@ private static SortOrder freshSortOrder(int orderId, Schema schema, SortOrder so } for (SortField field : sortOrder.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = sortOrder.schema().findColumnName(field.sourceId()); // reassign all sort fields with fresh sort field IDs - int newSourceId = schema.findField(sourceName).fieldId(); + List newSourceIds = freshSourceIds(sortOrder.schema(), schema, field.sourceIds()); builder.addSortField( - field.transform().toString(), newSourceId, field.direction(), field.nullOrder()); + field.transform().toString(), newSourceIds, field.direction(), field.nullOrder()); } return builder.build().bind(schema); } + private static List freshSourceIds( + Schema oldSchema, Schema newSchema, List sourceIds) { + List freshIds = Lists.newArrayListWithCapacity(sourceIds.size()); + for (int sourceId : sourceIds) { + // look up the name of the source field in the old schema to get the new schema's id + String sourceName = oldSchema.findColumnName(sourceId); + freshIds.add(newSchema.findField(sourceName).fieldId()); + } + + return freshIds; + } + private static Map indexAndValidateSnapshots( List snapshots, long lastSequenceNumber) { ImmutableMap.Builder builder = ImmutableMap.builder(); diff --git a/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java b/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java index 654823993776..26d834fd6aae 100644 --- a/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java @@ -152,8 +152,9 @@ public static Set orderPreservingSortedColumns(SortOrder sortOrder) { if (sortOrder == null) { return Collections.emptySet(); } else { + // the order of a multi-arg transform does not preserve the order of any single column return sortOrder.fields().stream() - .filter(f -> f.transform().preservesOrder()) + .filter(f -> f.sourceIds().size() == 1 && f.transform().preservesOrder()) .map(SortField::sourceId) .map(sid -> sortOrder.schema().findColumnName(sid)) .filter(Objects::nonNull) diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java index e97d2f98b416..8e7f17e92495 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java @@ -19,9 +19,16 @@ package org.apache.iceberg; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Projections; +import org.apache.iceberg.expressions.Projections.ProjectionEvaluator; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.JsonUtil; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -69,6 +76,75 @@ public void testToJsonForV1Table() { assertThat(PartitionSpecParser.toJson(table.spec(), true)).isEqualTo(expected); } + @TestTemplate + public void testToJsonWithMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testToJsonWithMultipleFieldsAndMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data_bucket") + .addField("truncate[10]", Arrays.asList(3, 7, 10), 1002, "multi_truncate") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"multi_truncate\",\n" + + " \"transform\" : \"truncate[10]\",\n" + + " \"source-ids\" : [ 3, 7, 10 ],\n" + + " \"field-id\" : 1002\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testToJsonWithSingleSourceId() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(0) + .addField("bucket[16]", Collections.singletonList(2), 1000, "data_bucket") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 0,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 2,\n" + + " \"field-id\" : 1000\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + @TestTemplate public void testFromJsonWithFieldId() { String specString = @@ -119,6 +195,285 @@ public void testFromJsonWithoutFieldId() { assertThat(spec.fields().get(1).fieldId()).isEqualTo(1001); } + @TestTemplate + public void testFromJsonWithSourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 1,\n" + + " \"source-ids\" : [ 1 ],\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 2,\n" + + " \"source-ids\" : [ 2 ],\n" + + " \"field-id\" : 1000\n" + + " } ]\n" + + "}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(table.schema(), specString); + + assertThat(spec.fields()).hasSize(2); + assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + assertThat(spec.fields().get(1).sourceId()).isEqualTo(2); + assertThat(spec.fields().get(1).fieldId()).isEqualTo(1000); + } + + @TestTemplate + public void testFromJsonWithSourceIdsOnly() { + // source-ids without source-id for forward compatibility + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(table.schema(), specString); + + assertThat(spec.fields()).hasSize(1); + assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + } + + @TestTemplate + public void testFromJsonWithMultipleSourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(unboundSpec.fields().get(0).partitionId()).isEqualTo(1001); + + // round-trip should preserve source-ids + String json = PartitionSpecParser.toJson(unboundSpec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + assertThat(json).doesNotContain("\"source-id\""); + + UnboundPartitionSpec roundTripped = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(roundTripped.fields().get(0).sourceIds()).containsExactly(1, 2); + } + + @TestTemplate + public void testFromJsonWithEmptySourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> PartitionSpecParser.fromJson(table.schema(), specString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + + @TestTemplate + public void testFromJsonWithSourceIdsNotAnArray() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : 1,\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> PartitionSpecParser.fromJson(table.schema(), specString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + + @TestTemplate + public void testFromJsonWithMultipleSourceIdsWithoutFieldId() { + // source-ids without field-id should auto-increment field ids + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ]\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).partitionId()).isNull(); + } + + @TestTemplate + public void testFromJsonSourceIdsPrecedenceOverSourceId() { + // when both source-id and source-ids are present, source-ids takes precedence + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 99,\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + // source-ids should be used, not source-id + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).sourceId()).isEqualTo(1); + } + + @TestTemplate + public void testToJsonCompactWithMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data") + .build(); + + String json = PartitionSpecParser.toJson(spec, false); + assertThat(json).contains("\"source-ids\":[1,2]"); + assertThat(json).doesNotContain("\"source-id\""); + } + + @TestTemplate + public void testToJsonMixedSingleAndMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Collections.singletonList(1), 1001, "id_bucket") + .addField("bucket[16]", Arrays.asList(1, 2), 1002, "id_data_bucket") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 1,\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"id_data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1002\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testRoundTripSingleSourceIdUsesSingularKey() { + // a single-element source-ids list should serialize as source-id (not source-ids) + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Collections.singletonList(1), 1001, "id_bucket") + .build(); + + String json = PartitionSpecParser.toJson(spec, true); + assertThat(json).contains("\"source-id\" : 1"); + assertThat(json).doesNotContain("\"source-ids\""); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1); + } + + @TestTemplate + public void testFromJsonFieldsWithMultipleSourceIds() { + // exercise the fromJsonFields path used by table metadata parsing + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + String fieldsJson = + "[{\"name\":\"id_data\",\"transform\":\"bucket[8]\"," + + "\"source-ids\":[1,2],\"field-id\":1001}]"; + + PartitionSpec spec = + JsonUtil.parse(fieldsJson, node -> PartitionSpecParser.fromJsonFields(schema, 1, node)); + assertThat(spec.specId()).isEqualTo(1); + assertThat(spec.fields()).hasSize(1); + assertThat(spec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + } + + @TestTemplate + void multiArgFieldDoesNotProjectFilters() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + String specJson = + "{\"spec-id\":1,\"fields\":[{\"name\":\"id_data\",\"transform\":\"bucket[8]\"," + + "\"source-ids\":[1,2],\"field-id\":1001}]}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(schema, specJson); + ProjectionEvaluator projection = Projections.inclusive(spec); + + assertThat(projection.project(Expressions.equal("id", 5))).isEqualTo(Expressions.alwaysTrue()); + assertThat(projection.project(Expressions.equal("data", "a"))) + .isEqualTo(Expressions.alwaysTrue()); + } + + @TestTemplate + public void testBoundSpecRoundTripPreservesSourceIds() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + // build an unbound spec with multi-arg source-ids, bind it, then round-trip through JSON + UnboundPartitionSpec unboundSpec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data_bucket") + .build(); + + PartitionSpec boundSpec = unboundSpec.bind(schema); + assertThat(boundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + + // round-trip through JSON via toUnbound + String json = PartitionSpecParser.toJson(boundSpec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2); + } + @TestTemplate public void testTransforms() { for (PartitionSpec spec : PartitionSpecTestBase.SPECS) { @@ -126,6 +481,34 @@ public void testTransforms() { } } + @TestTemplate + public void testTransformRoundTripWithMultipleSourceIds() { + // test various transform types with multiple source-ids + String[] transforms = {"bucket[16]", "identity", "truncate[10]"}; + + for (String transform : transforms) { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField(transform, Arrays.asList(1, 2, 7), 1001, "multi_field") + .build(); + + String json = PartitionSpecParser.toJson(spec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2, 7 ]"); + assertThat(json).doesNotContain("\"source-id\""); + assertThat(json).contains("\"transform\" : \"" + transform + "\""); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.specId()).isEqualTo(1); + assertThat(parsed.fields()).hasSize(1); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2, 7); + assertThat(parsed.fields().get(0).sourceId()).isEqualTo(1); + assertThat(parsed.fields().get(0).transformAsString()).isEqualTo(transform); + assertThat(parsed.fields().get(0).partitionId()).isEqualTo(1001); + assertThat(parsed.fields().get(0).name()).isEqualTo("multi_field"); + } + } + private static PartitionSpec roundTripJSON(PartitionSpec spec) { return PartitionSpecParser.fromJson( PartitionSpecTestBase.SCHEMA, PartitionSpecParser.toJson(spec)); diff --git a/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java b/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java index adcc130c2390..783e63e9f948 100644 --- a/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java +++ b/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java @@ -19,6 +19,8 @@ package org.apache.iceberg; import static org.apache.iceberg.NullOrder.NULLS_FIRST; +import static org.apache.iceberg.NullOrder.NULLS_LAST; +import static org.apache.iceberg.SortDirection.ASC; import static org.apache.iceberg.SortDirection.DESC; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -62,6 +64,113 @@ public void testUnknownTransforms() { assertThat(order.fields().get(0).nullOrder()).isEqualTo(NULLS_FIRST); } + @TestTemplate + public void testSourceIdsDeserialization() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + UnboundSortOrder order = SortOrderParser.fromJson(jsonString); + + assertThat(order.orderId()).isEqualTo(10); + assertThat(order.fields()).hasSize(1); + assertThat(order.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(order.fields().get(0).sourceId()).isEqualTo(1); + assertThat(order.fields().get(0).direction()).isEqualTo(ASC); + assertThat(order.fields().get(0).nullOrder()).isEqualTo(NULLS_LAST); + } + + @TestTemplate + public void testSourceIdsSerialization() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", Arrays.asList(1, 2), ASC, NULLS_LAST) + .build(); + + String json = SortOrderParser.toJson(order, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + assertThat(json).doesNotContain("\"source-id\""); + } + + @TestTemplate + public void testSingleSourceIdSerialization() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", 1, ASC, NULLS_LAST) + .build(); + + String json = SortOrderParser.toJson(order, true); + assertThat(json).contains("\"source-id\" : 1"); + assertThat(json).doesNotContain("\"source-ids\""); + } + + @TestTemplate + public void testSourceIdsRoundTrip() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", Arrays.asList(1, 2), ASC, NULLS_LAST) + .addSortField("identity", 3, DESC, NULLS_FIRST) + .build(); + + String json = SortOrderParser.toJson(order, true); + UnboundSortOrder parsed = SortOrderParser.fromJson(json); + + assertThat(parsed.orderId()).isEqualTo(10); + assertThat(parsed.fields()).hasSize(2); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(parsed.fields().get(0).direction()).isEqualTo(ASC); + assertThat(parsed.fields().get(1).sourceIds()).containsExactly(3); + assertThat(parsed.fields().get(1).direction()).isEqualTo(DESC); + } + + @TestTemplate + public void testSourceIdsPrecedenceOverSourceId() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 99,\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + UnboundSortOrder order = SortOrderParser.fromJson(jsonString); + + assertThat(order.fields()).hasSize(1); + assertThat(order.fields().get(0).sourceIds()).containsExactly(1, 2); + } + + @TestTemplate + public void testEmptySourceIdsArray() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> SortOrderParser.fromJson(jsonString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + @TestTemplate public void invalidSortDirection() { String jsonString = diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..55c85a988437 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -2298,4 +2298,94 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form assertThat(updated.currentSnapshot().timestampMillis()).isEqualTo(snapshotTs); assertThat(updated.lastUpdatedMillis()).isLessThan(snapshotTs); } + + @Test + void newTableMetadataReassignsMultiArgSourceIds() { + Schema schema = + new Schema( + Types.NestedField.required(10, "id", Types.IntegerType.get()), + Types.NestedField.required(20, "data", Types.StringType.get())); + List sourceIds = ImmutableList.of(10, 20); + + TableMetadata metadata = + TableMetadata.newTableMetadata( + schema, + multiArgSpec(schema, sourceIds), + multiArgSortOrder(schema, sourceIds), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, "3")); + + List freshSourceIds = multiArgSourceIds(metadata.schema()); + assertThat(freshSourceIds).doesNotContainAnyElementsOf(sourceIds); + assertThat(metadata.spec().fields().get(0).sourceIds()).isEqualTo(freshSourceIds); + assertThat(metadata.sortOrder().fields().get(0).sourceIds()).isEqualTo(freshSourceIds); + } + + @Test + void schemaUpdatePreservesMultiArgFields() { + TableMetadata base = multiArgTableMetadata(); + List columns = Lists.newArrayList(base.schema().columns()); + columns.add( + Types.NestedField.optional(base.lastColumnId() + 1, "extra", Types.StringType.get())); + + TableMetadata updated = + TableMetadata.buildFrom(base) + .setCurrentSchema(new Schema(columns), base.lastColumnId() + 1) + .build(); + + List expectedSourceIds = multiArgSourceIds(updated.schema()); + assertThat(updated.spec().fields().get(0).sourceIds()).isEqualTo(expectedSourceIds); + assertThat(updated.sortOrder().fields().get(0).sourceIds()).isEqualTo(expectedSourceIds); + + TableMetadata parsed = TableMetadataParser.fromJson(TableMetadataParser.toJson(updated)); + assertThat(parsed.spec()).isEqualTo(updated.spec()); + assertThat(parsed.sortOrder()).isEqualTo(updated.sortOrder()); + } + + @Test + void replacementPreservesMultiArgFields() { + TableMetadata base = multiArgTableMetadata(); + + TableMetadata replaced = + base.buildReplacement( + base.schema(), base.spec(), base.sortOrder(), TEST_LOCATION, ImmutableMap.of()); + + // the multi-arg partition field is matched on all of its source ids and keeps its field id + assertThat(replaced.spec().fields()).isEqualTo(base.spec().fields()); + assertThat(replaced.sortOrder().fields()).isEqualTo(base.sortOrder().fields()); + } + + private static TableMetadata multiArgTableMetadata() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + List sourceIds = multiArgSourceIds(schema); + + return TableMetadata.newTableMetadata( + schema, + multiArgSpec(schema, sourceIds), + multiArgSortOrder(schema, sourceIds), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, "3")); + } + + private static List multiArgSourceIds(Schema schema) { + return ImmutableList.of(schema.findField("id").fieldId(), schema.findField("data").fieldId()); + } + + private static PartitionSpec multiArgSpec(Schema schema, List sourceIds) { + return UnboundPartitionSpec.builder() + .addField("bucket[8]", sourceIds, 1000, "id_data_bucket") + .build() + .bind(schema); + } + + private static SortOrder multiArgSortOrder(Schema schema, List sourceIds) { + return UnboundSortOrder.builder() + .withOrderId(1) + .addSortField("bucket[16]", sourceIds, SortDirection.ASC, NullOrder.NULLS_FIRST) + .build() + .bind(schema); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java index 9ee696b1e347..7d7987f71cc1 100644 --- a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java @@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.TestTemplate; @@ -49,6 +50,11 @@ public class TestUpdatePartitionSpec extends TestBase { .day("ts") .bucket("id", 16, "shard") .build(); + private static final PartitionSpec MULTI_ARG = + UnboundPartitionSpec.builder() + .addField("bucket[8]", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .build() + .bind(SCHEMA); @TestTemplate public void testAddIdentityByName() { @@ -658,6 +664,45 @@ public void testRemoveAndUpdateWithDifferentTransformation() { } } + @TestTemplate + void addFieldPreservesMultiArgField() { + PartitionSpec updated = + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).addField("data").apply(); + + PartitionSpec expected = + UnboundPartitionSpec.builder() + .addField("bucket[8]", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .addField("identity", id("data"), 1001, "data") + .build() + .bind(SCHEMA); + + assertThat(updated).isEqualTo(expected); + } + + @TestTemplate + void removeMultiArgFieldByName() { + PartitionSpec updated = + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).removeField("id_category").apply(); + + PartitionSpec v1Expected = + UnboundPartitionSpec.builder() + .addField("void", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .build() + .bind(SCHEMA); + + V1Assert.assertEquals("Should match expected spec", v1Expected, updated); + V2Assert.assertEquals("Should match expected spec", UNPARTITIONED, updated); + } + + @TestTemplate + void singleArgTermDoesNotMatchMultiArgField() { + assertThatThrownBy( + () -> + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).removeField(bucket("id", 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Cannot find partition field to remove"); + } + private static int id(String name) { return SCHEMA.findField(name).fieldId(); }