From 4bf0eb558907cc77477acba8396458060b3b4426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 5 Mar 2026 07:26:05 -0500 Subject: [PATCH 01/11] Core: add source-ids support for multi-args transform --- .../org/apache/iceberg/PartitionField.java | 32 +- .../org/apache/iceberg/PartitionSpec.java | 75 ++-- .../java/org/apache/iceberg/SortField.java | 37 +- .../java/org/apache/iceberg/SortOrder.java | 51 ++- .../apache/iceberg/UnboundPartitionSpec.java | 47 ++- .../org/apache/iceberg/UnboundSortOrder.java | 42 +- .../apache/iceberg/PartitionSpecParser.java | 50 ++- .../org/apache/iceberg/SortOrderParser.java | 38 +- .../iceberg/TestPartitionSpecParser.java | 362 ++++++++++++++++++ .../apache/iceberg/TestSortOrderParser.java | 109 ++++++ 10 files changed, 758 insertions(+), 85 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionField.java b/api/src/main/java/org/apache/iceberg/PartitionField.java index 3ed765a89834..b1a9960d1dd9 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionField.java +++ b/api/src/main/java/org/apache/iceberg/PartitionField.java @@ -19,18 +19,25 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.base.Objects; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; /** Represents a single field in a {@link PartitionSpec}. */ public class PartitionField implements Serializable { - private final int sourceId; + private final List sourceIds; private final int fieldId; private final String name; private final Transform transform; PartitionField(int sourceId, int fieldId, String name, Transform transform) { - this.sourceId = sourceId; + this(ImmutableList.of(sourceId), fieldId, name, transform); + } + + PartitionField( + List sourceIds, int fieldId, String name, Transform transform) { + this.sourceIds = ImmutableList.copyOf(sourceIds); this.fieldId = fieldId; this.name = name; this.transform = transform; @@ -38,7 +45,17 @@ public class PartitionField implements Serializable { /** Returns the field id of the source field in the {@link PartitionSpec spec's} table schema. */ public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + /** + * Returns the field ids of all source fields for this partition field. + * + *

For single-argument transforms, this list contains one element. For multi-argument + * transforms, this list contains multiple source field ids. + */ + public List sourceIds() { + return sourceIds; } /** Returns the partition field id across all the table metadata's partition specs. */ @@ -58,7 +75,10 @@ public String name() { @Override public String toString() { - return fieldId + ": " + name + ": " + transform + "(" + sourceId + ")"; + if (sourceIds.size() == 1) { + return fieldId + ": " + name + ": " + transform + "(" + sourceIds.get(0) + ")"; + } + return fieldId + ": " + name + ": " + transform + "(" + sourceIds + ")"; } @Override @@ -70,7 +90,7 @@ public boolean equals(Object other) { } PartitionField that = (PartitionField) other; - return sourceId == that.sourceId + return sourceIds.equals(that.sourceIds) && fieldId == that.fieldId && name.equals(that.name) && transform.toString().equals(that.transform.toString()); @@ -78,6 +98,6 @@ public boolean equals(Object other) { @Override public int hashCode() { - return Objects.hashCode(sourceId, fieldId, name, transform); + return Objects.hashCode(sourceIds, fieldId, name, transform); } } diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index 9c1528582c8a..e07d391ac044 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -115,7 +115,7 @@ public UnboundPartitionSpec toUnbound() { for (PartitionField field : fields) { builder.addField( - field.transform().toString(), field.sourceId(), field.fieldId(), field.name()); + field.transform().toString(), field.sourceIds(), field.fieldId(), field.name()); } return builder.build(); @@ -254,7 +254,7 @@ public boolean compatibleWith(PartitionSpec other) { for (int i = 0; i < fields.length; i += 1) { PartitionField thisField = fields[i]; PartitionField thatField = other.fields[i]; - if (thisField.sourceId() != thatField.sourceId() + if (!thisField.sourceIds().equals(thatField.sourceIds()) || !thisField.transform().toString().equals(thatField.transform().toString()) || !thisField.name().equals(thatField.name())) { return false; @@ -627,6 +627,17 @@ Builder add(int sourceId, int fieldId, String name, Transform transform) { return this; } + Builder add(List sourceIds, int fieldId, String name, Transform transform) { + checkAndAddPartitionName(name, sourceIds.get(0)); + fields.add(new PartitionField(sourceIds, fieldId, name, transform)); + lastAssignedFieldId.getAndAccumulate(fieldId, Math::max); + return this; + } + + Builder add(List sourceIds, String name, Transform transform) { + return add(sourceIds, nextFieldId(), name, transform); + } + public PartitionSpec build() { return build(false); } @@ -649,37 +660,39 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) { static void checkCompatibility(PartitionSpec spec, Schema schema, boolean allowMissingFields) { final Map parents = TypeUtil.indexParents(schema.asStruct()); for (PartitionField field : spec.fields) { - Type sourceType = schema.findType(field.sourceId()); Transform transform = field.transform(); - // In the case the underlying field is dropped, we cannot check if they are compatible - if (allowMissingFields && sourceType == null) { - continue; - } - // In the case of a Version 1 partition-spec field gets deleted, - // it is replaced with a void transform, see: - // https://iceberg.apache.org/spec/#partition-transforms - // We don't care about the source type since a VoidTransform is always compatible and skip the - // checks - if (!transform.equals(Transforms.alwaysNull())) { - ValidationException.check( - sourceType != null, "Cannot find source column for partition field: %s", field); - ValidationException.check( - sourceType.isPrimitiveType(), - "Cannot partition by non-primitive source field: %s", - sourceType); - ValidationException.check( - transform.canTransform(sourceType), - "Invalid source type %s for transform: %s", - sourceType, - transform); - // The only valid parent types for a PartitionField are StructTypes. This must be checked - // recursively. - Integer parentId = parents.get(field.sourceId()); - while (parentId != null) { - Type parentType = schema.findType(parentId); + for (int srcId : field.sourceIds()) { + Type sourceType = schema.findType(srcId); + // In the case the underlying field is dropped, we cannot check if they are compatible + if (allowMissingFields && sourceType == null) { + continue; + } + // In the case of a Version 1 partition-spec field gets deleted, + // it is replaced with a void transform, see: + // https://iceberg.apache.org/spec/#partition-transforms + // We don't care about the source type since a VoidTransform is always compatible and skip + // the checks + if (!transform.equals(Transforms.alwaysNull())) { + ValidationException.check( + sourceType != null, "Cannot find source column for partition field: %s", field); ValidationException.check( - parentType.isStructType(), "Invalid partition field parent: %s", parentType); - parentId = parents.get(parentId); + sourceType.isPrimitiveType(), + "Cannot partition by non-primitive source field: %s", + sourceType); + ValidationException.check( + transform.canTransform(sourceType), + "Invalid source type %s for transform: %s", + sourceType, + transform); + // The only valid parent types for a PartitionField are StructTypes. This must be checked + // recursively. + Integer parentId = parents.get(srcId); + while (parentId != null) { + Type parentType = schema.findType(parentId); + ValidationException.check( + parentType.isStructType(), "Invalid partition field parent: %s", parentType); + parentId = parents.get(parentId); + } } } } diff --git a/api/src/main/java/org/apache/iceberg/SortField.java b/api/src/main/java/org/apache/iceberg/SortField.java index d7f110a26e3f..676740a935ae 100644 --- a/api/src/main/java/org/apache/iceberg/SortField.java +++ b/api/src/main/java/org/apache/iceberg/SortField.java @@ -19,20 +19,30 @@ package org.apache.iceberg; import java.io.Serializable; +import java.util.List; import java.util.Objects; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; /** A field in a {@link SortOrder}. */ public class SortField implements Serializable { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final SortDirection direction; private final NullOrder nullOrder; SortField(Transform transform, int sourceId, SortDirection direction, NullOrder nullOrder) { + this(transform, ImmutableList.of(sourceId), direction, nullOrder); + } + + SortField( + Transform transform, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { this.transform = transform; - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.direction = direction; this.nullOrder = nullOrder; } @@ -51,7 +61,17 @@ public Transform transform() { /** Returns the field id of the source field in the {@link SortOrder sort order's} table schema */ public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + /** + * Returns the field ids of all source fields for this sort field. + * + *

For single-argument transforms, this list contains one element. For multi-argument + * transforms, this list contains multiple source field ids. + */ + public List sourceIds() { + return sourceIds; } /** Returns the sort direction */ @@ -73,7 +93,7 @@ public NullOrder nullOrder() { public boolean satisfies(SortField other) { if (Objects.equals(this, other)) { return true; - } else if (sourceId != other.sourceId + } else if (!sourceIds.equals(other.sourceIds) || direction != other.direction || nullOrder != other.nullOrder) { return false; @@ -84,7 +104,10 @@ public boolean satisfies(SortField other) { @Override public String toString() { - return transform + "(" + sourceId + ") " + direction + " " + nullOrder; + if (sourceIds.size() == 1) { + return transform + "(" + sourceIds.get(0) + ") " + direction + " " + nullOrder; + } + return transform + "(" + sourceIds + ") " + direction + " " + nullOrder; } @Override @@ -97,13 +120,13 @@ public boolean equals(Object other) { SortField that = (SortField) other; return transform.toString().equals(that.transform.toString()) - && sourceId == that.sourceId + && sourceIds.equals(that.sourceIds) && direction == that.direction && nullOrder == that.nullOrder; } @Override public int hashCode() { - return Objects.hash(transform, sourceId, direction, nullOrder); + return Objects.hash(transform, sourceIds, direction, nullOrder); } } diff --git a/api/src/main/java/org/apache/iceberg/SortOrder.java b/api/src/main/java/org/apache/iceberg/SortOrder.java index e98e05bd1e88..f7662da9d265 100644 --- a/api/src/main/java/org/apache/iceberg/SortOrder.java +++ b/api/src/main/java/org/apache/iceberg/SortOrder.java @@ -126,8 +126,19 @@ public UnboundSortOrder toUnbound() { UnboundSortOrder.Builder builder = UnboundSortOrder.builder().withOrderId(orderId); for (SortField field : fields) { - builder.addSortField( - field.transform().toString(), field.sourceId(), field.direction(), field.nullOrder()); + if (field.sourceIds().size() > 1) { + builder.addSortField( + field.transform().toString(), + field.sourceIds(), + field.direction(), + field.nullOrder()); + } else { + builder.addSortField( + field.transform().toString(), + field.sourceId(), + field.direction(), + field.nullOrder()); + } } return builder.build(); @@ -260,6 +271,16 @@ Builder addSortField( return this; } + Builder addSortField( + Transform transform, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { + SortField sortField = new SortField(transform, sourceIds, direction, nullOrder); + fields.add(sortField); + return this; + } + public SortOrder build() { SortOrder sortOrder = buildUnchecked(); checkCompatibility(sortOrder, schema); @@ -297,18 +318,20 @@ SortOrder buildUnchecked() { public static void checkCompatibility(SortOrder sortOrder, Schema schema) { for (SortField field : sortOrder.fields) { - Type sourceType = schema.findType(field.sourceId()); - ValidationException.check( - sourceType != null, "Cannot find source column for sort field: %s", field); - ValidationException.check( - sourceType.isPrimitiveType(), - "Cannot sort by non-primitive source field: %s", - sourceType); - ValidationException.check( - field.transform().canTransform(sourceType), - "Invalid source type %s for transform: %s", - sourceType, - field.transform()); + for (int srcId : field.sourceIds()) { + Type sourceType = schema.findType(srcId); + ValidationException.check( + sourceType != null, "Cannot find source column for sort field: %s", field); + ValidationException.check( + sourceType.isPrimitiveType(), + "Cannot sort by non-primitive source field: %s", + sourceType); + ValidationException.check( + field.transform().canTransform(sourceType), + "Invalid source type %s for transform: %s", + sourceType, + field.transform()); + } } } } diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java index 30b3cce35fc8..27e4ae3d0e47 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java @@ -19,6 +19,7 @@ package org.apache.iceberg; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; @@ -58,17 +59,25 @@ private PartitionSpec.Builder copyToBuilder(Schema schema) { PartitionSpec.Builder builder = PartitionSpec.builderFor(schema).withSpecId(specId); for (UnboundPartitionField field : fields) { - Type fieldType = schema.findType(field.sourceId); + Type fieldType = schema.findType(field.sourceId()); Transform transform; if (fieldType != null) { - transform = Transforms.fromString(fieldType, field.transform.toString()); + transform = Transforms.fromString(fieldType, field.transformAsString()); } else { - transform = Transforms.fromString(field.transform.toString()); + transform = Transforms.fromString(field.transformAsString()); } - if (field.partitionId != null) { - builder.add(field.sourceId, field.partitionId, field.name, transform); + if (field.sourceIds().size() > 1) { + if (field.partitionId() != null) { + builder.add(field.sourceIds(), field.partitionId(), field.name(), transform); + } else { + builder.add(field.sourceIds(), field.name(), transform); + } } else { - builder.add(field.sourceId, field.name, transform); + if (field.partitionId() != null) { + builder.add(field.sourceId(), field.partitionId(), field.name(), transform); + } else { + builder.add(field.sourceId(), field.name(), transform); + } } } @@ -102,6 +111,17 @@ Builder addField(String transformAsString, int sourceId, String name) { return this; } + Builder addField( + String transformAsString, List sourceIds, int partitionId, String name) { + fields.add(new UnboundPartitionField(transformAsString, sourceIds, partitionId, name)); + return this; + } + + Builder addField(String transformAsString, List sourceIds, String name) { + fields.add(new UnboundPartitionField(transformAsString, sourceIds, null, name)); + return this; + } + UnboundPartitionSpec build() { return new UnboundPartitionSpec(specId, fields); } @@ -109,7 +129,7 @@ UnboundPartitionSpec build() { static class UnboundPartitionField { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final Integer partitionId; private final String name; @@ -122,7 +142,11 @@ public String transformAsString() { } public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + public List sourceIds() { + return sourceIds; } public Integer partitionId() { @@ -135,8 +159,13 @@ public String name() { private UnboundPartitionField( String transformAsString, int sourceId, Integer partitionId, String name) { + this(transformAsString, ImmutableList.of(sourceId), partitionId, name); + } + + private UnboundPartitionField( + String transformAsString, List sourceIds, Integer partitionId, String name) { this.transform = Transforms.fromString(transformAsString); - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.partitionId = partitionId; this.name = name; } diff --git a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java index ce9f6b1d6b2c..57a1b95182a0 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java +++ b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java @@ -20,6 +20,7 @@ import java.util.Collections; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.Transforms; @@ -41,14 +42,18 @@ public SortOrder bind(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - Type sourceType = schema.findType(field.sourceId); + Type sourceType = schema.findType(field.sourceId()); Transform transform; if (sourceType != null) { transform = Transforms.fromString(sourceType, field.transform.toString()); } else { transform = field.transform; } - builder.addSortField(transform, field.sourceId, field.direction, field.nullOrder); + if (field.sourceIds().size() > 1) { + builder.addSortField(transform, field.sourceIds(), field.direction, field.nullOrder); + } else { + builder.addSortField(transform, field.sourceId(), field.direction, field.nullOrder); + } } return builder.build(); @@ -58,7 +63,11 @@ SortOrder bindUnchecked(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder); + if (field.sourceIds().size() > 1) { + builder.addSortField(field.transform, field.sourceIds(), field.direction, field.nullOrder); + } else { + builder.addSortField(field.transform, field.sourceId(), field.direction, field.nullOrder); + } } return builder.buildUnchecked(); @@ -103,6 +112,15 @@ Builder addSortField( return this; } + Builder addSortField( + String transformAsString, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { + fields.add(new UnboundSortField(transformAsString, sourceIds, direction, nullOrder)); + return this; + } + UnboundSortOrder build() { if (fields.isEmpty()) { if (orderId != null && orderId != 0) { @@ -123,14 +141,22 @@ UnboundSortOrder build() { static class UnboundSortField { private final Transform transform; - private final int sourceId; + private final List sourceIds; private final SortDirection direction; private final NullOrder nullOrder; private UnboundSortField( String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) { + this(transformAsString, ImmutableList.of(sourceId), direction, nullOrder); + } + + private UnboundSortField( + String transformAsString, + List sourceIds, + SortDirection direction, + NullOrder nullOrder) { this.transform = Transforms.fromString(transformAsString); - this.sourceId = sourceId; + this.sourceIds = ImmutableList.copyOf(sourceIds); this.direction = direction; this.nullOrder = nullOrder; } @@ -140,7 +166,11 @@ public String transformAsString() { } public int sourceId() { - return sourceId; + return sourceIds.get(0); + } + + public List sourceIds() { + return sourceIds; } public SortDirection direction() { diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index 7becf0c62943..bccd45a59ae6 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -24,7 +24,9 @@ import com.github.benmanes.caffeine.cache.Caffeine; import java.io.IOException; import java.util.Iterator; +import java.util.List; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; import org.apache.iceberg.util.Pair; @@ -35,6 +37,7 @@ private PartitionSpecParser() {} private static final String SPEC_ID = "spec-id"; private static final String FIELDS = "fields"; private static final String SOURCE_ID = "source-id"; + private static final String SOURCE_IDS = "source-ids"; private static final String FIELD_ID = "field-id"; private static final String TRANSFORM = "transform"; private static final String NAME = "name"; @@ -98,7 +101,16 @@ static void toJsonFields(UnboundPartitionSpec spec, JsonGenerator generator) thr generator.writeStartObject(); generator.writeStringField(NAME, field.name()); generator.writeStringField(TRANSFORM, field.transformAsString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + if (field.sourceIds().size() == 1) { + generator.writeNumberField(SOURCE_ID, field.sourceId()); + } else { + generator.writeFieldName(SOURCE_IDS); + generator.writeStartArray(); + for (Integer value : field.sourceIds()) { + generator.writeNumber(value); + } + generator.writeEndArray(); + } generator.writeNumberField(FIELD_ID, field.partitionId()); generator.writeEndObject(); } @@ -132,15 +144,37 @@ private static void buildFromJsonFields(UnboundPartitionSpec.Builder builder, Js String name = JsonUtil.getString(NAME, element); String transform = JsonUtil.getString(TRANSFORM, element); - int sourceId = JsonUtil.getInt(SOURCE_ID, element); - // partition field ids are missing in old PartitionSpec, they always auto-increment from - // PARTITION_DATA_ID_START - if (element.has(FIELD_ID)) { - builder.addField(transform, sourceId, JsonUtil.getInt(FIELD_ID, element), name); - fieldIdCount++; + if (element.has(SOURCE_IDS)) { + JsonNode sourceIdsNode = element.get(SOURCE_IDS); + Preconditions.checkArgument( + sourceIdsNode.isArray() && sourceIdsNode.size() > 0, + "Cannot parse partition field, source-ids must be a non-empty array: %s", + element); + List sourceIds = Lists.newArrayListWithCapacity(sourceIdsNode.size()); + for (int i = 0; i < sourceIdsNode.size(); i++) { + sourceIds.add(sourceIdsNode.get(i).asInt()); + } + + // partition field ids are missing in old PartitionSpec, they always auto-increment from + // PARTITION_DATA_ID_START + if (element.has(FIELD_ID)) { + builder.addField(transform, sourceIds, JsonUtil.getInt(FIELD_ID, element), name); + fieldIdCount++; + } else { + builder.addField(transform, sourceIds, name); + } } else { - builder.addField(transform, sourceId, name); + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + + // partition field ids are missing in old PartitionSpec, they always auto-increment from + // PARTITION_DATA_ID_START + if (element.has(FIELD_ID)) { + builder.addField(transform, sourceId, JsonUtil.getInt(FIELD_ID, element), name); + fieldIdCount++; + } else { + builder.addField(transform, sourceId, name); + } } } diff --git a/core/src/main/java/org/apache/iceberg/SortOrderParser.java b/core/src/main/java/org/apache/iceberg/SortOrderParser.java index 74126162aaf2..8edc75cfcdab 100644 --- a/core/src/main/java/org/apache/iceberg/SortOrderParser.java +++ b/core/src/main/java/org/apache/iceberg/SortOrderParser.java @@ -25,8 +25,10 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import java.util.Iterator; +import java.util.List; import java.util.Locale; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.util.JsonUtil; public class SortOrderParser { @@ -36,6 +38,7 @@ public class SortOrderParser { private static final String NULL_ORDER = "null-order"; private static final String TRANSFORM = "transform"; private static final String SOURCE_ID = "source-id"; + private static final String SOURCE_IDS = "source-ids"; private SortOrderParser() {} @@ -69,7 +72,7 @@ private static void toJsonFields(SortOrder sortOrder, JsonGenerator generator) for (SortField field : sortOrder.fields()) { generator.writeStartObject(); generator.writeStringField(TRANSFORM, field.transform().toString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + writeSourceIds(field.sourceIds(), generator); generator.writeStringField(DIRECTION, toJson(field.direction())); generator.writeStringField(NULL_ORDER, toJson(field.nullOrder())); generator.writeEndObject(); @@ -100,7 +103,7 @@ private static void toJsonFields(UnboundSortOrder sortOrder, JsonGenerator gener for (UnboundSortOrder.UnboundSortField field : sortOrder.fields()) { generator.writeStartObject(); generator.writeStringField(TRANSFORM, field.transformAsString()); - generator.writeNumberField(SOURCE_ID, field.sourceId()); + writeSourceIds(field.sourceIds(), generator); generator.writeStringField(DIRECTION, toJson(field.direction())); generator.writeStringField(NULL_ORDER, toJson(field.nullOrder())); generator.writeEndObject(); @@ -108,6 +111,20 @@ private static void toJsonFields(UnboundSortOrder sortOrder, JsonGenerator gener generator.writeEndArray(); } + private static void writeSourceIds(List sourceIds, JsonGenerator generator) + throws IOException { + if (sourceIds.size() == 1) { + generator.writeNumberField(SOURCE_ID, sourceIds.get(0)); + } else { + generator.writeFieldName(SOURCE_IDS); + generator.writeStartArray(); + for (Integer value : sourceIds) { + generator.writeNumber(value); + } + generator.writeEndArray(); + } + } + public static SortOrder fromJson(Schema schema, String json) { return fromJson(json).bind(schema); } @@ -155,7 +172,6 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo element.isObject(), "Cannot parse sort field, not an object: %s", element); String transform = JsonUtil.getString(TRANSFORM, element); - int sourceId = JsonUtil.getInt(SOURCE_ID, element); String directionAsString = JsonUtil.getString(DIRECTION, element); SortDirection direction = SortDirection.fromString(directionAsString); @@ -163,7 +179,21 @@ private static void buildFromJsonFields(UnboundSortOrder.Builder builder, JsonNo String nullOrderingAsString = JsonUtil.getString(NULL_ORDER, element); NullOrder nullOrder = toNullOrder(nullOrderingAsString); - builder.addSortField(transform, sourceId, direction, nullOrder); + if (element.has(SOURCE_IDS)) { + JsonNode sourceIdsNode = element.get(SOURCE_IDS); + Preconditions.checkArgument( + sourceIdsNode.isArray() && sourceIdsNode.size() > 0, + "Cannot parse sort field, source-ids must be a non-empty array: %s", + element); + List sourceIds = Lists.newArrayListWithCapacity(sourceIdsNode.size()); + for (int i = 0; i < sourceIdsNode.size(); i++) { + sourceIds.add(sourceIdsNode.get(i).asInt()); + } + builder.addSortField(transform, sourceIds, direction, nullOrder); + } else { + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + builder.addSortField(transform, sourceId, direction, nullOrder); + } } } diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java index e97d2f98b416..870eea984425 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java @@ -19,9 +19,13 @@ package org.apache.iceberg; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.JsonUtil; import org.junit.jupiter.api.TestTemplate; import org.junit.jupiter.api.extension.ExtendWith; @@ -69,6 +73,75 @@ public void testToJsonForV1Table() { assertThat(PartitionSpecParser.toJson(table.spec(), true)).isEqualTo(expected); } + @TestTemplate + public void testToJsonWithMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testToJsonWithMultipleFieldsAndMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data_bucket") + .addField("truncate[10]", Arrays.asList(3, 7, 10), 1002, "multi_truncate") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"multi_truncate\",\n" + + " \"transform\" : \"truncate[10]\",\n" + + " \"source-ids\" : [ 3, 7, 10 ],\n" + + " \"field-id\" : 1002\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testToJsonWithSingleSourceId() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(0) + .addField("bucket[16]", Collections.singletonList(2), 1000, "data_bucket") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 0,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 2,\n" + + " \"field-id\" : 1000\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + @TestTemplate public void testFromJsonWithFieldId() { String specString = @@ -119,6 +192,267 @@ public void testFromJsonWithoutFieldId() { assertThat(spec.fields().get(1).fieldId()).isEqualTo(1001); } + @TestTemplate + public void testFromJsonWithSourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 1,\n" + + " \"source-ids\" : [ 1 ],\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 2,\n" + + " \"source-ids\" : [ 2 ],\n" + + " \"field-id\" : 1000\n" + + " } ]\n" + + "}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(table.schema(), specString); + + assertThat(spec.fields()).hasSize(2); + assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + assertThat(spec.fields().get(1).sourceId()).isEqualTo(2); + assertThat(spec.fields().get(1).fieldId()).isEqualTo(1000); + } + + @TestTemplate + public void testFromJsonWithSourceIdsOnly() { + // source-ids without source-id for forward compatibility + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(table.schema(), specString); + + assertThat(spec.fields()).hasSize(1); + assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + } + + @TestTemplate + public void testFromJsonWithMultipleSourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(unboundSpec.fields().get(0).partitionId()).isEqualTo(1001); + + // round-trip should preserve source-ids + String json = PartitionSpecParser.toJson(unboundSpec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + assertThat(json).doesNotContain("\"source-id\""); + + UnboundPartitionSpec roundTripped = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(roundTripped.fields().get(0).sourceIds()).containsExactly(1, 2); + } + + @TestTemplate + public void testFromJsonWithEmptySourceIds() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> PartitionSpecParser.fromJson(table.schema(), specString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + + @TestTemplate + public void testFromJsonWithSourceIdsNotAnArray() { + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : 1,\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> PartitionSpecParser.fromJson(table.schema(), specString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + + @TestTemplate + public void testFromJsonWithMultipleSourceIdsWithoutFieldId() { + // source-ids without field-id should auto-increment field ids + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-ids\" : [ 1, 2 ]\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).partitionId()).isNull(); + } + + @TestTemplate + public void testFromJsonSourceIdsPrecedenceOverSourceId() { + // when both source-id and source-ids are present, source-ids takes precedence + String specString = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_data\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 99,\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1001\n" + + " } ]\n" + + "}"; + + UnboundPartitionSpec unboundSpec = JsonUtil.parse(specString, PartitionSpecParser::fromJson); + + assertThat(unboundSpec.fields()).hasSize(1); + // source-ids should be used, not source-id + assertThat(unboundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(unboundSpec.fields().get(0).sourceId()).isEqualTo(1); + } + + @TestTemplate + public void testToJsonCompactWithMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data") + .build(); + + String json = PartitionSpecParser.toJson(spec, false); + assertThat(json).contains("\"source-ids\":[1,2]"); + assertThat(json).doesNotContain("\"source-id\""); + } + + @TestTemplate + public void testToJsonMixedSingleAndMultipleSourceIds() { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Collections.singletonList(1), 1001, "id_bucket") + .addField("bucket[16]", Arrays.asList(1, 2), 1002, "id_data_bucket") + .build(); + + String expected = + "{\n" + + " \"spec-id\" : 1,\n" + + " \"fields\" : [ {\n" + + " \"name\" : \"id_bucket\",\n" + + " \"transform\" : \"bucket[8]\",\n" + + " \"source-id\" : 1,\n" + + " \"field-id\" : 1001\n" + + " }, {\n" + + " \"name\" : \"id_data_bucket\",\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"field-id\" : 1002\n" + + " } ]\n" + + "}"; + assertThat(PartitionSpecParser.toJson(spec, true)).isEqualTo(expected); + } + + @TestTemplate + public void testRoundTripSingleSourceIdUsesSingularKey() { + // a single-element source-ids list should serialize as source-id (not source-ids) + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Collections.singletonList(1), 1001, "id_bucket") + .build(); + + String json = PartitionSpecParser.toJson(spec, true); + assertThat(json).contains("\"source-id\" : 1"); + assertThat(json).doesNotContain("\"source-ids\""); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1); + } + + @TestTemplate + public void testFromJsonFieldsWithMultipleSourceIds() { + // exercise the fromJsonFields path used by table metadata parsing + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + String fieldsJson = + "[{\"name\":\"id_data\",\"transform\":\"bucket[8]\"," + + "\"source-ids\":[1,2],\"field-id\":1001}]"; + + PartitionSpec spec = + JsonUtil.parse(fieldsJson, node -> PartitionSpecParser.fromJsonFields(schema, 1, node)); + assertThat(spec.specId()).isEqualTo(1); + assertThat(spec.fields()).hasSize(1); + assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); + assertThat(spec.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); + } + + @TestTemplate + public void testBoundSpecRoundTripPreservesSourceIds() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + // build an unbound spec with multi-arg source-ids, bind it, then round-trip through JSON + UnboundPartitionSpec unboundSpec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField("bucket[8]", Arrays.asList(1, 2), 1001, "id_data_bucket") + .build(); + + PartitionSpec boundSpec = unboundSpec.bind(schema); + assertThat(boundSpec.fields().get(0).sourceIds()).containsExactly(1, 2); + + // round-trip through JSON via toUnbound + String json = PartitionSpecParser.toJson(boundSpec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2); + } + @TestTemplate public void testTransforms() { for (PartitionSpec spec : PartitionSpecTestBase.SPECS) { @@ -126,6 +460,34 @@ public void testTransforms() { } } + @TestTemplate + public void testTransformRoundTripWithMultipleSourceIds() { + // test various transform types with multiple source-ids + String[] transforms = {"bucket[16]", "identity", "truncate[10]"}; + + for (String transform : transforms) { + UnboundPartitionSpec spec = + UnboundPartitionSpec.builder() + .withSpecId(1) + .addField(transform, Arrays.asList(1, 2, 7), 1001, "multi_field") + .build(); + + String json = PartitionSpecParser.toJson(spec, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2, 7 ]"); + assertThat(json).doesNotContain("\"source-id\""); + assertThat(json).contains("\"transform\" : \"" + transform + "\""); + + UnboundPartitionSpec parsed = JsonUtil.parse(json, PartitionSpecParser::fromJson); + assertThat(parsed.specId()).isEqualTo(1); + assertThat(parsed.fields()).hasSize(1); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2, 7); + assertThat(parsed.fields().get(0).sourceId()).isEqualTo(1); + assertThat(parsed.fields().get(0).transformAsString()).isEqualTo(transform); + assertThat(parsed.fields().get(0).partitionId()).isEqualTo(1001); + assertThat(parsed.fields().get(0).name()).isEqualTo("multi_field"); + } + } + private static PartitionSpec roundTripJSON(PartitionSpec spec) { return PartitionSpecParser.fromJson( PartitionSpecTestBase.SCHEMA, PartitionSpecParser.toJson(spec)); diff --git a/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java b/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java index adcc130c2390..783e63e9f948 100644 --- a/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java +++ b/core/src/test/java/org/apache/iceberg/TestSortOrderParser.java @@ -19,6 +19,8 @@ package org.apache.iceberg; import static org.apache.iceberg.NullOrder.NULLS_FIRST; +import static org.apache.iceberg.NullOrder.NULLS_LAST; +import static org.apache.iceberg.SortDirection.ASC; import static org.apache.iceberg.SortDirection.DESC; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -62,6 +64,113 @@ public void testUnknownTransforms() { assertThat(order.fields().get(0).nullOrder()).isEqualTo(NULLS_FIRST); } + @TestTemplate + public void testSourceIdsDeserialization() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + UnboundSortOrder order = SortOrderParser.fromJson(jsonString); + + assertThat(order.orderId()).isEqualTo(10); + assertThat(order.fields()).hasSize(1); + assertThat(order.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(order.fields().get(0).sourceId()).isEqualTo(1); + assertThat(order.fields().get(0).direction()).isEqualTo(ASC); + assertThat(order.fields().get(0).nullOrder()).isEqualTo(NULLS_LAST); + } + + @TestTemplate + public void testSourceIdsSerialization() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", Arrays.asList(1, 2), ASC, NULLS_LAST) + .build(); + + String json = SortOrderParser.toJson(order, true); + assertThat(json).contains("\"source-ids\" : [ 1, 2 ]"); + assertThat(json).doesNotContain("\"source-id\""); + } + + @TestTemplate + public void testSingleSourceIdSerialization() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", 1, ASC, NULLS_LAST) + .build(); + + String json = SortOrderParser.toJson(order, true); + assertThat(json).contains("\"source-id\" : 1"); + assertThat(json).doesNotContain("\"source-ids\""); + } + + @TestTemplate + public void testSourceIdsRoundTrip() { + UnboundSortOrder order = + UnboundSortOrder.builder() + .withOrderId(10) + .addSortField("bucket[16]", Arrays.asList(1, 2), ASC, NULLS_LAST) + .addSortField("identity", 3, DESC, NULLS_FIRST) + .build(); + + String json = SortOrderParser.toJson(order, true); + UnboundSortOrder parsed = SortOrderParser.fromJson(json); + + assertThat(parsed.orderId()).isEqualTo(10); + assertThat(parsed.fields()).hasSize(2); + assertThat(parsed.fields().get(0).sourceIds()).containsExactly(1, 2); + assertThat(parsed.fields().get(0).direction()).isEqualTo(ASC); + assertThat(parsed.fields().get(1).sourceIds()).containsExactly(3); + assertThat(parsed.fields().get(1).direction()).isEqualTo(DESC); + } + + @TestTemplate + public void testSourceIdsPrecedenceOverSourceId() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-id\" : 99,\n" + + " \"source-ids\" : [ 1, 2 ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + UnboundSortOrder order = SortOrderParser.fromJson(jsonString); + + assertThat(order.fields()).hasSize(1); + assertThat(order.fields().get(0).sourceIds()).containsExactly(1, 2); + } + + @TestTemplate + public void testEmptySourceIdsArray() { + String jsonString = + "{\n" + + " \"order-id\" : 10,\n" + + " \"fields\" : [ {\n" + + " \"transform\" : \"bucket[16]\",\n" + + " \"source-ids\" : [ ],\n" + + " \"direction\" : \"asc\",\n" + + " \"null-order\" : \"nulls-last\"\n" + + " } ]\n" + + "}"; + + assertThatThrownBy(() -> SortOrderParser.fromJson(jsonString)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source-ids must be a non-empty array"); + } + @TestTemplate public void invalidSortDirection() { String jsonString = From 5afd382414438820e7d0a23210161c671bd5dd9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 6 Mar 2026 14:54:17 +0100 Subject: [PATCH 02/11] Core: accept revapi breaks for PartitionField and SortField serialization changes --- .palantir/revapi.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 81d5fa6072a0..eaabffc2a2db 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -1040,6 +1040,15 @@ acceptedBreaks: justification: "Removing deprecated createStructReader overload scheduled for\ \ removal in 1.12.0" "1.12.0": + org.apache.iceberg:iceberg-api: + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.PartitionField" + new: "class org.apache.iceberg.PartitionField" + justification: "Adding source-ids field to support multi-arg transforms" + - code: "java.class.defaultSerializationChanged" + old: "class org.apache.iceberg.SortField" + new: "class org.apache.iceberg.SortField" + justification: "Adding source-ids field to support multi-arg transforms" org.apache.iceberg:iceberg-core: - code: "java.field.removedWithConstant" old: "field org.apache.iceberg.rest.RESTCatalogProperties.SIGNER_ENDPOINT" From e0844f0d9b1d95c6540d166a968cbbbb5e515275 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 6 Mar 2026 14:58:33 +0100 Subject: [PATCH 03/11] Core: fix spotless formatting --- .../main/java/org/apache/iceberg/PartitionField.java | 3 +-- api/src/main/java/org/apache/iceberg/SortOrder.java | 10 ++-------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionField.java b/api/src/main/java/org/apache/iceberg/PartitionField.java index b1a9960d1dd9..ac9e46b9b6c9 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionField.java +++ b/api/src/main/java/org/apache/iceberg/PartitionField.java @@ -35,8 +35,7 @@ public class PartitionField implements Serializable { this(ImmutableList.of(sourceId), fieldId, name, transform); } - PartitionField( - List sourceIds, int fieldId, String name, Transform transform) { + PartitionField(List sourceIds, int fieldId, String name, Transform transform) { this.sourceIds = ImmutableList.copyOf(sourceIds); this.fieldId = fieldId; this.name = name; diff --git a/api/src/main/java/org/apache/iceberg/SortOrder.java b/api/src/main/java/org/apache/iceberg/SortOrder.java index f7662da9d265..52127a26856d 100644 --- a/api/src/main/java/org/apache/iceberg/SortOrder.java +++ b/api/src/main/java/org/apache/iceberg/SortOrder.java @@ -128,16 +128,10 @@ public UnboundSortOrder toUnbound() { for (SortField field : fields) { if (field.sourceIds().size() > 1) { builder.addSortField( - field.transform().toString(), - field.sourceIds(), - field.direction(), - field.nullOrder()); + field.transform().toString(), field.sourceIds(), field.direction(), field.nullOrder()); } else { builder.addSortField( - field.transform().toString(), - field.sourceId(), - field.direction(), - field.nullOrder()); + field.transform().toString(), field.sourceId(), field.direction(), field.nullOrder()); } } From 6ecdb10fddcd5069a50372701b405573baa221b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 13 Mar 2026 14:34:37 +0100 Subject: [PATCH 04/11] Core: harden PartitionField and PartitionSpec for multi-arg transforms Add a precondition guard on PartitionField.sourceId() to prevent misuse with multi-arg transforms, fix toString() to use Joiner instead of List.toString(), migrate all sourceId() call sites in PartitionSpec to use sourceIds(), and add a consistency check in PartitionSpecParser when both source-id and source-ids are present. --- .../org/apache/iceberg/PartitionField.java | 13 ++++++--- .../org/apache/iceberg/PartitionSpec.java | 28 +++++++++++-------- .../apache/iceberg/PartitionSpecParser.java | 10 +++++++ 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionField.java b/api/src/main/java/org/apache/iceberg/PartitionField.java index ac9e46b9b6c9..119cc86586ac 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionField.java +++ b/api/src/main/java/org/apache/iceberg/PartitionField.java @@ -20,12 +20,16 @@ import java.io.Serializable; import java.util.List; +import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.base.Objects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; /** Represents a single field in a {@link PartitionSpec}. */ public class PartitionField implements Serializable { + private static final Joiner COMMA = Joiner.on(", "); + private final List sourceIds; private final int fieldId; private final String name; @@ -44,6 +48,10 @@ public class PartitionField implements Serializable { /** Returns the field id of the source field in the {@link PartitionSpec spec's} table schema. */ public int sourceId() { + Preconditions.checkState( + sourceIds.size() == 1, + "Use sourceIds() for multi-argument transforms, found %s source ids", + sourceIds.size()); return sourceIds.get(0); } @@ -74,10 +82,7 @@ public String name() { @Override public String toString() { - if (sourceIds.size() == 1) { - return fieldId + ": " + name + ": " + transform + "(" + sourceIds.get(0) + ")"; - } - return fieldId + ": " + name + ": " + transform + "(" + sourceIds + ")"; + return fieldId + ": " + name + ": " + transform + "(" + COMMA.join(sourceIds) + ")"; } @Override diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index e07d391ac044..3670982299b2 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -198,7 +198,7 @@ public Class[] javaClasses() { } private Type resultType(PartitionField field) { - Type sourceType = schema.findType(field.sourceId()); + Type sourceType = schema.findType(field.sourceIds().get(0)); if (sourceType == null) { // When the source field has been dropped, the source type has been lost // Transforms with a fixed result type still work, so use unknown for source @@ -303,7 +303,9 @@ private ListMultimap lazyFieldsBySourceId() { Multimaps.newListMultimap( Maps.newHashMap(), () -> Lists.newArrayListWithCapacity(fields.length)); for (PartitionField field : fields) { - multiMap.put(field.sourceId(), field); + for (int sourceId : field.sourceIds()) { + multiMap.put(sourceId, field); + } } this.fieldsBySourceId = multiMap; } @@ -322,7 +324,7 @@ public Set identitySourceIds() { Set sourceIds = Sets.newHashSet(); for (PartitionField field : fields()) { if (field.transform().isIdentity()) { - sourceIds.add(field.sourceId()); + sourceIds.addAll(field.sourceIds()); } } @@ -433,15 +435,17 @@ private void checkAndAddPartitionName(String name, Integer sourceColumnId) { } private void checkForRedundantPartitions(PartitionField field) { - Map.Entry dedupKey = - new AbstractMap.SimpleEntry<>(field.sourceId(), field.transform().dedupName()); - PartitionField partitionField = dedupFields.get(dedupKey); - Preconditions.checkArgument( - partitionField == null, - "Cannot add redundant partition: %s conflicts with %s", - partitionField, - field); - dedupFields.put(dedupKey, field); + for (int sourceId : field.sourceIds()) { + Map.Entry dedupKey = + new AbstractMap.SimpleEntry<>(sourceId, field.transform().dedupName()); + PartitionField partitionField = dedupFields.get(dedupKey); + Preconditions.checkArgument( + partitionField == null, + "Cannot add redundant partition: %s conflicts with %s", + partitionField, + field); + dedupFields.put(dedupKey, field); + } } public Builder caseSensitive(boolean sensitive) { diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index bccd45a59ae6..fa59b4ab6bc5 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -156,6 +156,16 @@ private static void buildFromJsonFields(UnboundPartitionSpec.Builder builder, Js sourceIds.add(sourceIdsNode.get(i).asInt()); } + if (element.has(SOURCE_ID)) { + int sourceId = JsonUtil.getInt(SOURCE_ID, element); + Preconditions.checkArgument( + sourceIds.get(0).equals(sourceId), + "Conflicting source-id (%s) and source-ids (%s) in partition field: %s", + sourceId, + sourceIds, + element); + } + // partition field ids are missing in old PartitionSpec, they always auto-increment from // PARTITION_DATA_ID_START if (element.has(FIELD_ID)) { From 51ec373f5960985ee62380e68c53c953567e6283 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 13 Mar 2026 14:41:03 +0100 Subject: [PATCH 05/11] Core: use LOG.warn instead of precondition for conflicting source-id/source-ids Follow reviewer guidance to warn rather than throw when parsing, since Iceberg generally avoids hard failures during deserialization. --- .../org/apache/iceberg/PartitionSpecParser.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index fa59b4ab6bc5..39202758db2d 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -30,8 +30,12 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; import org.apache.iceberg.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class PartitionSpecParser { + private static final Logger LOG = LoggerFactory.getLogger(PartitionSpecParser.class); + private PartitionSpecParser() {} private static final String SPEC_ID = "spec-id"; @@ -158,12 +162,13 @@ private static void buildFromJsonFields(UnboundPartitionSpec.Builder builder, Js if (element.has(SOURCE_ID)) { int sourceId = JsonUtil.getInt(SOURCE_ID, element); - Preconditions.checkArgument( - sourceIds.get(0).equals(sourceId), - "Conflicting source-id (%s) and source-ids (%s) in partition field: %s", - sourceId, - sourceIds, - element); + if (!sourceIds.get(0).equals(sourceId)) { + LOG.warn( + "Conflicting source-id ({}) and source-ids ({}) in partition field: {}", + sourceId, + sourceIds, + element); + } } // partition field ids are missing in old PartitionSpec, they always auto-increment from From 281463887c740204981e9d4f8d93c1f163e5c79d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 13 Mar 2026 15:18:37 +0100 Subject: [PATCH 06/11] Core: fix test calling sourceId() on multi-arg partition field The testFromJsonFieldsWithMultipleSourceIds test was calling sourceId() on a field with multiple source IDs, which now throws after the hardening in 7a4cc928b. Use sourceIds() instead. --- .../test/java/org/apache/iceberg/TestPartitionSpecParser.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java index 870eea984425..5c41f861cc47 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java @@ -423,7 +423,6 @@ public void testFromJsonFieldsWithMultipleSourceIds() { JsonUtil.parse(fieldsJson, node -> PartitionSpecParser.fromJsonFields(schema, 1, node)); assertThat(spec.specId()).isEqualTo(1); assertThat(spec.fields()).hasSize(1); - assertThat(spec.fields().get(0).sourceId()).isEqualTo(1); assertThat(spec.fields().get(0).sourceIds()).containsExactly(1, 2); assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); } From ea1ddfebdb2848b14f8ace44e4aad8bdbc69b828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Fri, 13 Mar 2026 18:26:46 +0100 Subject: [PATCH 07/11] Trigger CI re-run From 9bac292d51f839c257ff48a6c174f9f8a6f9f3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Mon, 16 Mar 2026 08:43:57 +0100 Subject: [PATCH 08/11] Trigger CI re-run From b52da2b1a3bbd4796cfc98fb356d889921e88785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 10 Nov 2026 10:47:51 +0100 Subject: [PATCH 09/11] Core: preserve source-ids when rebuilding partition specs and sort orders Table metadata maintenance rebuilt partition and sort fields from sourceId(), so a table with a multi-arg field could not evolve its schema, add a spec, or be replaced, and multi-arg sort fields were silently truncated to their first source id and written back to metadata. Rebuild fields from sourceIds() in TableMetadata, Partitioning and BaseUpdatePartitionSpec, and make SortField.sourceId() fail fast for multi-arg fields like PartitionField.sourceId() so remaining callers cannot silently use only the first argument. --- .../java/org/apache/iceberg/SortField.java | 5 ++ .../org/apache/iceberg/TestSortField.java | 59 ++++++++++++ .../iceberg/BaseUpdatePartitionSpec.java | 18 ++-- .../java/org/apache/iceberg/Partitioning.java | 16 +++- .../org/apache/iceberg/TableMetadata.java | 59 +++++++----- .../apache/iceberg/util/SortOrderUtil.java | 3 +- .../org/apache/iceberg/TestPartitioning.java | 28 ++++++ .../org/apache/iceberg/TestTableMetadata.java | 90 +++++++++++++++++++ .../iceberg/TestUpdatePartitionSpec.java | 45 ++++++++++ 9 files changed, 288 insertions(+), 35 deletions(-) create mode 100644 api/src/test/java/org/apache/iceberg/TestSortField.java diff --git a/api/src/main/java/org/apache/iceberg/SortField.java b/api/src/main/java/org/apache/iceberg/SortField.java index 676740a935ae..b3dab01abc81 100644 --- a/api/src/main/java/org/apache/iceberg/SortField.java +++ b/api/src/main/java/org/apache/iceberg/SortField.java @@ -21,6 +21,7 @@ import java.io.Serializable; import java.util.List; import java.util.Objects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transform; @@ -61,6 +62,10 @@ public Transform transform() { /** Returns the field id of the source field in the {@link SortOrder sort order's} table schema */ public int sourceId() { + Preconditions.checkState( + sourceIds.size() == 1, + "Use sourceIds() for multi-argument transforms, found %s source ids", + sourceIds.size()); return sourceIds.get(0); } diff --git a/api/src/test/java/org/apache/iceberg/TestSortField.java b/api/src/test/java/org/apache/iceberg/TestSortField.java new file mode 100644 index 000000000000..27dc59fc4261 --- /dev/null +++ b/api/src/test/java/org/apache/iceberg/TestSortField.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.required; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestSortField { + private static final Schema SCHEMA = + new Schema( + required(1, "id", Types.IntegerType.get()), required(2, "data", Types.StringType.get())); + + @Test + void sourceIdForSingleArgField() { + SortField field = SortOrder.builderFor(SCHEMA).asc("id").build().fields().get(0); + + assertThat(field.sourceIds()).containsExactly(1); + assertThat(field.sourceId()).isEqualTo(1); + } + + @Test + void sourceIdFailsForMultiArgField() { + SortField field = + UnboundSortOrder.builder() + .withOrderId(1) + .addSortField( + "bucket[16]", ImmutableList.of(1, 2), SortDirection.ASC, NullOrder.NULLS_FIRST) + .build() + .bind(SCHEMA) + .fields() + .get(0); + + assertThat(field.sourceIds()).containsExactly(1, 2); + assertThatThrownBy(field::sourceId) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Use sourceIds() for multi-argument transforms, found 2 source ids"); + } +} diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java index c69f6f3844f9..bcb97f74a222 100644 --- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java +++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java @@ -30,6 +30,7 @@ import org.apache.iceberg.expressions.UnboundTerm; import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -122,7 +123,7 @@ private int assignFieldId() { private PartitionField recycleOrCreatePartitionField( Pair> sourceTransform, String name) { if (formatVersion >= 2 && base != null) { - int sourceId = sourceTransform.first(); + List sourceIds = ImmutableList.of(sourceTransform.first()); Transform transform = sourceTransform.second(); Set allHistoricalFields = Sets.newHashSet(); @@ -131,7 +132,7 @@ private PartitionField recycleOrCreatePartitionField( } for (PartitionField field : allHistoricalFields) { - if (field.sourceId() == sourceId && field.transform().equals(transform)) { + if (field.sourceIds().equals(sourceIds) && field.transform().equals(transform)) { // if target name is specified then consider it too, otherwise not if (name == null || field.name().equals(name)) { return field; @@ -308,9 +309,9 @@ public PartitionSpec apply() { if (!deletes.contains(field.fieldId())) { String newName = renames.get(field.name()); if (newName != null) { - builder.add(field.sourceId(), field.fieldId(), newName, field.transform()); + builder.add(field.sourceIds(), field.fieldId(), newName, field.transform()); } else { - builder.add(field.sourceId(), field.fieldId(), field.name(), field.transform()); + builder.add(field.sourceIds(), field.fieldId(), field.name(), field.transform()); } } else if (formatVersion < 2) { // field IDs were not required for v1 and were assigned sequentially in each partition spec @@ -320,9 +321,9 @@ public PartitionSpec apply() { // must be replaced with a null transform. null values are always allowed in partition data. String newName = renames.get(field.name()); if (newName != null) { - builder.add(field.sourceId(), field.fieldId(), newName, Transforms.alwaysNull()); + builder.add(field.sourceIds(), field.fieldId(), newName, Transforms.alwaysNull()); } else { - builder.add(field.sourceId(), field.fieldId(), field.name(), Transforms.alwaysNull()); + builder.add(field.sourceIds(), field.fieldId(), field.name(), Transforms.alwaysNull()); } } } @@ -399,7 +400,10 @@ private static Map, PartitionField> indexSpecByTransform( Map, PartitionField> indexSpecs = Maps.newHashMap(); List fields = spec.fields(); for (PartitionField field : fields) { - indexSpecs.put(Pair.of(field.sourceId(), field.transform().toString()), field); + // multi-arg fields cannot be referenced by a term, which always has a single source field + if (field.sourceIds().size() == 1) { + indexSpecs.put(Pair.of(field.sourceId(), field.transform().toString()), field); + } } return indexSpecs; diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index 66a681f88486..b0df5b53aadb 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -348,7 +348,7 @@ private static boolean isVoidTransform(PartitionField field) { private static boolean equivalentIgnoringNames( PartitionField field, PartitionField anotherField) { return field.fieldId() == anotherField.fieldId() - && field.sourceId() == anotherField.sourceId() + && field.sourceIds().equals(anotherField.sourceIds()) && compatibleTransforms(field.transform(), anotherField.transform()); } @@ -370,7 +370,7 @@ private static Set allFieldIds(Collection specs) { private static Set allActiveFieldIds(Schema schema, Collection specs) { return FluentIterable.from(specs) .transformAndConcat(PartitionSpec::fields) - .filter(field -> schema.findField(field.sourceId()) != null) + .filter(field -> hasAllSourceFields(schema, field)) .transform(PartitionField::fieldId) .toSet(); } @@ -395,9 +395,19 @@ private static Set commonActiveFieldIds(Schema schema, Collection activeFieldIds(Schema schema, PartitionSpec spec) { return spec.fields().stream() - .filter(field -> schema == null || schema.findField(field.sourceId()) != null) + .filter(field -> schema == null || hasAllSourceFields(schema, field)) .filter(field -> !isVoidTransform(field)) .map(PartitionField::fieldId) .collect(Collectors.toList()); } + + private static boolean hasAllSourceFields(Schema schema, PartitionField field) { + for (int sourceId : field.sourceIds()) { + if (schema.findField(sourceId) == null) { + return false; + } + } + + return true; + } } diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 89427e4c6943..35f36c81bc57 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -127,10 +127,9 @@ static TableMetadata newTableMetadata( PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(freshSchema).withSpecId(INITIAL_SPEC_ID); for (PartitionField field : spec.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = schema.findColumnName(field.sourceId()); // reassign all partition fields with fresh partition field Ids to ensure consistency - specBuilder.add(freshSchema.findField(sourceName).fieldId(), field.name(), field.transform()); + specBuilder.add( + freshSourceIds(schema, freshSchema, field.sourceIds()), field.name(), field.transform()); } PartitionSpec freshSpec = specBuilder.build(); @@ -652,12 +651,12 @@ private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil if (formatVersion > 1) { // for v2 and later, reuse any existing field IDs, but reproduce the same spec - Map, Integer> transformToFieldId = + Map, String>, Integer> transformToFieldId = specs.stream() .flatMap(spec -> spec.fields().stream()) .collect( Collectors.toMap( - field -> Pair.of(field.sourceId(), field.transform().toString()), + field -> Pair.of(field.sourceIds(), field.transform().toString()), PartitionField::fieldId, Math::max)); @@ -665,8 +664,8 @@ private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil // reassign the partition field ids int partitionFieldId = transformToFieldId.computeIfAbsent( - Pair.of(field.sourceId(), field.transform().toString()), k -> nextID.get()); - specBuilder.add(field.sourceId(), partitionFieldId, field.name(), field.transform()); + Pair.of(field.sourceIds(), field.transform().toString()), k -> nextID.get()); + specBuilder.add(field.sourceIds(), partitionFieldId, field.name(), field.transform()); } } else { @@ -763,7 +762,7 @@ private static PartitionSpec updateSpecSchema(Schema schema, PartitionSpec parti // add all the fields to the builder. IDs should not change. for (PartitionField field : partitionSpec.fields()) { - specBuilder.add(field.sourceId(), field.fieldId(), field.name(), field.transform()); + specBuilder.add(field.sourceIds(), field.fieldId(), field.name(), field.transform()); } // build without validation because the schema may have changed in a way that makes this spec @@ -778,7 +777,7 @@ private static SortOrder updateSortOrderSchema(Schema schema, SortOrder sortOrde // add all the fields to the builder. IDs should not change. for (SortField field : sortOrder.fields()) { builder.addSortField( - field.transform(), field.sourceId(), field.direction(), field.nullOrder()); + field.transform(), field.sourceIds(), field.direction(), field.nullOrder()); } // build without validation because the schema may have changed in a way that makes this order @@ -791,18 +790,20 @@ private static PartitionSpec freshSpec(int specId, Schema schema, PartitionSpec UnboundPartitionSpec.Builder specBuilder = UnboundPartitionSpec.builder().withSpecId(specId); for (PartitionField field : partitionSpec.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = partitionSpec.schema().findColumnName(field.sourceId()); - - final int fieldId; - if (sourceName != null) { - fieldId = schema.findField(sourceName).fieldId(); - } else { - // In the case of a null sourceName, the column has been deleted. - // This only happens in V1 tables where the reference is still around as a void transform - fieldId = field.sourceId(); + List sourceIds = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + // look up the name of the source field in the old schema to get the new schema's id + String sourceName = partitionSpec.schema().findColumnName(sourceId); + if (sourceName != null) { + sourceIds.add(schema.findField(sourceName).fieldId()); + } else { + // In the case of a null sourceName, the column has been deleted. + // This only happens in V1 tables where the reference is still around as a void transform + sourceIds.add(sourceId); + } } - specBuilder.addField(field.transform().toString(), fieldId, field.fieldId(), field.name()); + + specBuilder.addField(field.transform().toString(), sourceIds, field.fieldId(), field.name()); } return specBuilder.build().bind(schema); @@ -816,17 +817,27 @@ private static SortOrder freshSortOrder(int orderId, Schema schema, SortOrder so } for (SortField field : sortOrder.fields()) { - // look up the name of the source field in the old schema to get the new schema's id - String sourceName = sortOrder.schema().findColumnName(field.sourceId()); // reassign all sort fields with fresh sort field IDs - int newSourceId = schema.findField(sourceName).fieldId(); + List newSourceIds = freshSourceIds(sortOrder.schema(), schema, field.sourceIds()); builder.addSortField( - field.transform().toString(), newSourceId, field.direction(), field.nullOrder()); + field.transform().toString(), newSourceIds, field.direction(), field.nullOrder()); } return builder.build().bind(schema); } + private static List freshSourceIds( + Schema oldSchema, Schema newSchema, List sourceIds) { + List freshIds = Lists.newArrayListWithCapacity(sourceIds.size()); + for (int sourceId : sourceIds) { + // look up the name of the source field in the old schema to get the new schema's id + String sourceName = oldSchema.findColumnName(sourceId); + freshIds.add(newSchema.findField(sourceName).fieldId()); + } + + return freshIds; + } + private static Map indexAndValidateSnapshots( List snapshots, long lastSequenceNumber) { ImmutableMap.Builder builder = ImmutableMap.builder(); diff --git a/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java b/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java index 654823993776..26d834fd6aae 100644 --- a/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/SortOrderUtil.java @@ -152,8 +152,9 @@ public static Set orderPreservingSortedColumns(SortOrder sortOrder) { if (sortOrder == null) { return Collections.emptySet(); } else { + // the order of a multi-arg transform does not preserve the order of any single column return sortOrder.fields().stream() - .filter(f -> f.transform().preservesOrder()) + .filter(f -> f.sourceIds().size() == 1 && f.transform().preservesOrder()) .map(SortField::sourceId) .map(sid -> sortOrder.schema().findColumnName(sid)) .filter(Objects::nonNull) diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 4c98d66c10f7..b721d315bd85 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -25,6 +25,7 @@ import java.io.File; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StructType; @@ -36,6 +37,7 @@ public class TestPartitioning { private static final int V1_FORMAT_VERSION = 1; private static final int V2_FORMAT_VERSION = 2; + private static final int V3_FORMAT_VERSION = 3; private static final Schema SCHEMA = new Schema( required(1, "id", Types.IntegerType.get()), @@ -569,4 +571,30 @@ public void deleteFileAfterDeletingOnePartitionField() { table.newDelete().deleteFile(dataFile).commit(); assertThat(table.currentSnapshot().summary()).containsEntry("deleted-data-files", "1"); } + + @Test + void partitionTypeWithMultiArgFieldAndSpecEvolution() { + PartitionSpec multiArgSpec = + UnboundPartitionSpec.builder() + .addField("bucket[8]", ImmutableList.of(1, 2), 1000, "id_data_bucket") + .build() + .bind(SCHEMA); + TestTables.TestTable table = + TestTables.create(tableDir, "test", SCHEMA, multiArgSpec, V3_FORMAT_VERSION); + + table.updateSpec().addField("category").commit(); + + assertThat(table.specs()).hasSize(2); + + StructType expectedType = + StructType.of( + NestedField.optional(1000, "id_data_bucket", Types.IntegerType.get()), + NestedField.optional(1001, "category", Types.StringType.get())); + assertThat(Partitioning.partitionType(table)).isEqualTo(expectedType); + + StructType expectedGroupingKeyType = + StructType.of(NestedField.optional(1000, "id_data_bucket", Types.IntegerType.get())); + assertThat(Partitioning.groupingKeyType(table.schema(), table.specs().values())) + .isEqualTo(expectedGroupingKeyType); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..55c85a988437 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -2298,4 +2298,94 @@ public void snapshotTimestampMayBeAheadOfTableMetadataLastUpdatedMillis(int form assertThat(updated.currentSnapshot().timestampMillis()).isEqualTo(snapshotTs); assertThat(updated.lastUpdatedMillis()).isLessThan(snapshotTs); } + + @Test + void newTableMetadataReassignsMultiArgSourceIds() { + Schema schema = + new Schema( + Types.NestedField.required(10, "id", Types.IntegerType.get()), + Types.NestedField.required(20, "data", Types.StringType.get())); + List sourceIds = ImmutableList.of(10, 20); + + TableMetadata metadata = + TableMetadata.newTableMetadata( + schema, + multiArgSpec(schema, sourceIds), + multiArgSortOrder(schema, sourceIds), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, "3")); + + List freshSourceIds = multiArgSourceIds(metadata.schema()); + assertThat(freshSourceIds).doesNotContainAnyElementsOf(sourceIds); + assertThat(metadata.spec().fields().get(0).sourceIds()).isEqualTo(freshSourceIds); + assertThat(metadata.sortOrder().fields().get(0).sourceIds()).isEqualTo(freshSourceIds); + } + + @Test + void schemaUpdatePreservesMultiArgFields() { + TableMetadata base = multiArgTableMetadata(); + List columns = Lists.newArrayList(base.schema().columns()); + columns.add( + Types.NestedField.optional(base.lastColumnId() + 1, "extra", Types.StringType.get())); + + TableMetadata updated = + TableMetadata.buildFrom(base) + .setCurrentSchema(new Schema(columns), base.lastColumnId() + 1) + .build(); + + List expectedSourceIds = multiArgSourceIds(updated.schema()); + assertThat(updated.spec().fields().get(0).sourceIds()).isEqualTo(expectedSourceIds); + assertThat(updated.sortOrder().fields().get(0).sourceIds()).isEqualTo(expectedSourceIds); + + TableMetadata parsed = TableMetadataParser.fromJson(TableMetadataParser.toJson(updated)); + assertThat(parsed.spec()).isEqualTo(updated.spec()); + assertThat(parsed.sortOrder()).isEqualTo(updated.sortOrder()); + } + + @Test + void replacementPreservesMultiArgFields() { + TableMetadata base = multiArgTableMetadata(); + + TableMetadata replaced = + base.buildReplacement( + base.schema(), base.spec(), base.sortOrder(), TEST_LOCATION, ImmutableMap.of()); + + // the multi-arg partition field is matched on all of its source ids and keeps its field id + assertThat(replaced.spec().fields()).isEqualTo(base.spec().fields()); + assertThat(replaced.sortOrder().fields()).isEqualTo(base.sortOrder().fields()); + } + + private static TableMetadata multiArgTableMetadata() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + List sourceIds = multiArgSourceIds(schema); + + return TableMetadata.newTableMetadata( + schema, + multiArgSpec(schema, sourceIds), + multiArgSortOrder(schema, sourceIds), + TEST_LOCATION, + ImmutableMap.of(TableProperties.FORMAT_VERSION, "3")); + } + + private static List multiArgSourceIds(Schema schema) { + return ImmutableList.of(schema.findField("id").fieldId(), schema.findField("data").fieldId()); + } + + private static PartitionSpec multiArgSpec(Schema schema, List sourceIds) { + return UnboundPartitionSpec.builder() + .addField("bucket[8]", sourceIds, 1000, "id_data_bucket") + .build() + .bind(schema); + } + + private static SortOrder multiArgSortOrder(Schema schema, List sourceIds) { + return UnboundSortOrder.builder() + .withOrderId(1) + .addSortField("bucket[16]", sourceIds, SortDirection.ASC, NullOrder.NULLS_FIRST) + .build() + .bind(schema); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java index 9ee696b1e347..7d7987f71cc1 100644 --- a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java @@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.TestTemplate; @@ -49,6 +50,11 @@ public class TestUpdatePartitionSpec extends TestBase { .day("ts") .bucket("id", 16, "shard") .build(); + private static final PartitionSpec MULTI_ARG = + UnboundPartitionSpec.builder() + .addField("bucket[8]", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .build() + .bind(SCHEMA); @TestTemplate public void testAddIdentityByName() { @@ -658,6 +664,45 @@ public void testRemoveAndUpdateWithDifferentTransformation() { } } + @TestTemplate + void addFieldPreservesMultiArgField() { + PartitionSpec updated = + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).addField("data").apply(); + + PartitionSpec expected = + UnboundPartitionSpec.builder() + .addField("bucket[8]", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .addField("identity", id("data"), 1001, "data") + .build() + .bind(SCHEMA); + + assertThat(updated).isEqualTo(expected); + } + + @TestTemplate + void removeMultiArgFieldByName() { + PartitionSpec updated = + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).removeField("id_category").apply(); + + PartitionSpec v1Expected = + UnboundPartitionSpec.builder() + .addField("void", ImmutableList.of(id("id"), id("category")), 1000, "id_category") + .build() + .bind(SCHEMA); + + V1Assert.assertEquals("Should match expected spec", v1Expected, updated); + V2Assert.assertEquals("Should match expected spec", UNPARTITIONED, updated); + } + + @TestTemplate + void singleArgTermDoesNotMatchMultiArgField() { + assertThatThrownBy( + () -> + new BaseUpdatePartitionSpec(formatVersion, MULTI_ARG).removeField(bucket("id", 8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageStartingWith("Cannot find partition field to remove"); + } + private static int id(String name) { return SCHEMA.findField(name).fieldId(); } From c9ce75f6d80683e349a7892c84425448dd2c81cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 10 Nov 2026 11:42:45 +0100 Subject: [PATCH 10/11] API: reject multi-arg partition names that match a schema column A multi-arg partition field cannot be an identity partition, so its name must not collide with any schema column, including one of its sources. --- .../org/apache/iceberg/PartitionSpec.java | 7 ++++- .../iceberg/TestPartitionSpecValidation.java | 11 ++++++++ .../org/apache/iceberg/TestPartitioning.java | 28 ------------------- 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index 3670982299b2..cb1e01862fe8 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -632,7 +632,12 @@ Builder add(int sourceId, int fieldId, String name, Transform transform) { } Builder add(List sourceIds, int fieldId, String name, Transform transform) { - checkAndAddPartitionName(name, sourceIds.get(0)); + if (sourceIds.size() == 1) { + checkAndAddPartitionName(name, sourceIds.get(0)); + } else { + checkAndAddPartitionName(name); + } + fields.add(new PartitionField(sourceIds, fieldId, name, transform)); lastAssignedFieldId.getAndAccumulate(fieldId, Math::max); return this; diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index 5c37ad527291..0185f1ca6328 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -22,6 +22,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import org.apache.iceberg.exceptions.ValidationException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; @@ -242,6 +243,16 @@ public void testSettingPartitionTransformsWithCustomTargetNamesThatAlreadyExist( "Cannot create identity partition sourced from different field in schema: another_ts"); } + @Test + void multiArgPartitionNameMatchingSourceColumn() { + assertThatThrownBy( + () -> + PartitionSpec.builderFor(SCHEMA) + .add(ImmutableList.of(1, 6), "id", Transforms.bucket(8))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot create partition from name that exists in schema: id"); + } + @Test public void testStalePartitionSourceIdWithReusedColumnName() { int newFieldId = 2; diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index b721d315bd85..4c98d66c10f7 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -25,7 +25,6 @@ import java.io.File; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.apache.iceberg.types.Types.StructType; @@ -37,7 +36,6 @@ public class TestPartitioning { private static final int V1_FORMAT_VERSION = 1; private static final int V2_FORMAT_VERSION = 2; - private static final int V3_FORMAT_VERSION = 3; private static final Schema SCHEMA = new Schema( required(1, "id", Types.IntegerType.get()), @@ -571,30 +569,4 @@ public void deleteFileAfterDeletingOnePartitionField() { table.newDelete().deleteFile(dataFile).commit(); assertThat(table.currentSnapshot().summary()).containsEntry("deleted-data-files", "1"); } - - @Test - void partitionTypeWithMultiArgFieldAndSpecEvolution() { - PartitionSpec multiArgSpec = - UnboundPartitionSpec.builder() - .addField("bucket[8]", ImmutableList.of(1, 2), 1000, "id_data_bucket") - .build() - .bind(SCHEMA); - TestTables.TestTable table = - TestTables.create(tableDir, "test", SCHEMA, multiArgSpec, V3_FORMAT_VERSION); - - table.updateSpec().addField("category").commit(); - - assertThat(table.specs()).hasSize(2); - - StructType expectedType = - StructType.of( - NestedField.optional(1000, "id_data_bucket", Types.IntegerType.get()), - NestedField.optional(1001, "category", Types.StringType.get())); - assertThat(Partitioning.partitionType(table)).isEqualTo(expectedType); - - StructType expectedGroupingKeyType = - StructType.of(NestedField.optional(1000, "id_data_bucket", Types.IntegerType.get())); - assertThat(Partitioning.groupingKeyType(table.schema(), table.specs().values())) - .isEqualTo(expectedGroupingKeyType); - } } From 0f808d25c4290129b4c5923e088b6335d07f4cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Tue, 10 Nov 2026 11:42:45 +0100 Subject: [PATCH 11/11] API: bind multi-arg partition and sort fields as unknown transforms Multi-arg fields were bound using only the first source type, so a multi-arg bucket written by another engine was treated as a single-column bucket and used to prune files. Resolve transforms from all source types; since no multi-arg transform is implemented yet, they bind as unknown and are never used for filtering. --- .../apache/iceberg/UnboundPartitionSpec.java | 11 ++++---- .../org/apache/iceberg/UnboundSortOrder.java | 11 ++++---- .../apache/iceberg/transforms/Transforms.java | 25 +++++++++++++++++++ .../iceberg/TestPartitionSpecParser.java | 22 ++++++++++++++++ 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java index 27e4ae3d0e47..d359c46439f8 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java @@ -59,13 +59,12 @@ private PartitionSpec.Builder copyToBuilder(Schema schema) { PartitionSpec.Builder builder = PartitionSpec.builderFor(schema).withSpecId(specId); for (UnboundPartitionField field : fields) { - Type fieldType = schema.findType(field.sourceId()); - Transform transform; - if (fieldType != null) { - transform = Transforms.fromString(fieldType, field.transformAsString()); - } else { - transform = Transforms.fromString(field.transformAsString()); + List sourceTypes = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + sourceTypes.add(schema.findType(sourceId)); } + + Transform transform = Transforms.fromString(sourceTypes, field.transformAsString()); if (field.sourceIds().size() > 1) { if (field.partitionId() != null) { builder.add(field.sourceIds(), field.partitionId(), field.name(), transform); diff --git a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java index 57a1b95182a0..f815c438d5c1 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java +++ b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java @@ -42,13 +42,12 @@ public SortOrder bind(Schema schema) { SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId); for (UnboundSortField field : fields) { - Type sourceType = schema.findType(field.sourceId()); - Transform transform; - if (sourceType != null) { - transform = Transforms.fromString(sourceType, field.transform.toString()); - } else { - transform = field.transform; + List sourceTypes = Lists.newArrayListWithCapacity(field.sourceIds().size()); + for (int sourceId : field.sourceIds()) { + sourceTypes.add(schema.findType(sourceId)); } + + Transform transform = Transforms.fromString(sourceTypes, field.transformAsString()); if (field.sourceIds().size() > 1) { builder.addSortField(transform, field.sourceIds(), field.direction, field.nullOrder); } else { diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java index 958abd27375a..ed02cd0ea584 100644 --- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java +++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java @@ -18,11 +18,13 @@ */ package org.apache.iceberg.transforms; +import java.util.List; import java.util.Locale; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Type; /** @@ -115,6 +117,29 @@ private Transforms() {} return new UnknownTransform<>(transform); } + /** + * Parses a transform from its string representation for the given source types. + * + *

Multi-argument transforms are not recognized and produce an {@link UnknownTransform}. A + * {@code null} source type, for a source field that no longer exists, produces the same result as + * {@link #fromString(String)}. + * + * @param types the source types of the transform arguments, in order + * @param transform a transform string, such as {@code "day"} or {@code "bucket[16]"} + * @return the parsed transform + */ + public static Transform fromString(List types, String transform) { + Preconditions.checkArgument(types != null, "Invalid source types: null"); + Preconditions.checkArgument(!types.isEmpty(), "Invalid source types: empty"); + + if (types.size() > 1) { + return new UnknownTransform<>(transform); + } + + Type type = types.get(0); + return type != null ? fromString(type, transform) : fromString(transform); + } + /** * Returns an identity {@link Transform} that can be used for any type. * diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java index 5c41f861cc47..8e7f17e92495 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java @@ -24,6 +24,9 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Projections; +import org.apache.iceberg.expressions.Projections.ProjectionEvaluator; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.JsonUtil; import org.junit.jupiter.api.TestTemplate; @@ -427,6 +430,25 @@ public void testFromJsonFieldsWithMultipleSourceIds() { assertThat(spec.fields().get(0).fieldId()).isEqualTo(1001); } + @TestTemplate + void multiArgFieldDoesNotProjectFilters() { + Schema schema = + new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.required(2, "data", Types.StringType.get())); + + String specJson = + "{\"spec-id\":1,\"fields\":[{\"name\":\"id_data\",\"transform\":\"bucket[8]\"," + + "\"source-ids\":[1,2],\"field-id\":1001}]}"; + + PartitionSpec spec = PartitionSpecParser.fromJson(schema, specJson); + ProjectionEvaluator projection = Projections.inclusive(spec); + + assertThat(projection.project(Expressions.equal("id", 5))).isEqualTo(Expressions.alwaysTrue()); + assertThat(projection.project(Expressions.equal("data", "a"))) + .isEqualTo(Expressions.alwaysTrue()); + } + @TestTemplate public void testBoundSpecRoundTripPreservesSourceIds() { Schema schema =