Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .palantir/revapi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1040,6 +1040,15 @@ acceptedBreaks:
justification: "Removing deprecated createStructReader overload scheduled for\
\ removal in 1.12.0"
"1.12.0":
org.apache.iceberg:iceberg-api:
- code: "java.class.defaultSerializationChanged"
old: "class org.apache.iceberg.PartitionField"
new: "class org.apache.iceberg.PartitionField"
justification: "Adding source-ids field to support multi-arg transforms"
- code: "java.class.defaultSerializationChanged"
old: "class org.apache.iceberg.SortField"
new: "class org.apache.iceberg.SortField"
justification: "Adding source-ids field to support multi-arg transforms"
org.apache.iceberg:iceberg-core:
- code: "java.field.removedWithConstant"
old: "field org.apache.iceberg.rest.RESTCatalogProperties.SIGNER_ENDPOINT"
Expand Down
36 changes: 30 additions & 6 deletions api/src/main/java/org/apache/iceberg/PartitionField.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,26 +19,50 @@
package org.apache.iceberg;

import java.io.Serializable;
import java.util.List;
import org.apache.iceberg.relocated.com.google.common.base.Joiner;
import org.apache.iceberg.relocated.com.google.common.base.Objects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.transforms.Transform;

/** Represents a single field in a {@link PartitionSpec}. */
public class PartitionField implements Serializable {
private final int sourceId;
private static final Joiner COMMA = Joiner.on(", ");

private final List<Integer> sourceIds;
private final int fieldId;
private final String name;
private final Transform<?, ?> transform;

PartitionField(int sourceId, int fieldId, String name, Transform<?, ?> transform) {
this.sourceId = sourceId;
this(ImmutableList.of(sourceId), fieldId, name, transform);
}

PartitionField(List<Integer> sourceIds, int fieldId, String name, Transform<?, ?> transform) {
this.sourceIds = ImmutableList.copyOf(sourceIds);
this.fieldId = fieldId;
this.name = name;
this.transform = transform;
}

/** Returns the field id of the source field in the {@link PartitionSpec spec's} table schema. */
public int sourceId() {
return sourceId;
Preconditions.checkState(
sourceIds.size() == 1,
"Use sourceIds() for multi-argument transforms, found %s source ids",
sourceIds.size());
return sourceIds.get(0);
Comment thread
jbonofre marked this conversation as resolved.
Resolved
}

/**
* Returns the field ids of all source fields for this partition field.
*
* <p>For single-argument transforms, this list contains one element. For multi-argument
* transforms, this list contains multiple source field ids.
*/
public List<Integer> sourceIds() {
return sourceIds;
}

/** Returns the partition field id across all the table metadata's partition specs. */
Expand All @@ -58,7 +82,7 @@ public String name() {

@Override
public String toString() {
return fieldId + ": " + name + ": " + transform + "(" + sourceId + ")";
return fieldId + ": " + name + ": " + transform + "(" + COMMA.join(sourceIds) + ")";
}

@Override
Expand All @@ -70,14 +94,14 @@ public boolean equals(Object other) {
}

PartitionField that = (PartitionField) other;
return sourceId == that.sourceId
return sourceIds.equals(that.sourceIds)
&& fieldId == that.fieldId
&& name.equals(that.name)
&& transform.toString().equals(that.transform.toString());
}

@Override
public int hashCode() {
return Objects.hashCode(sourceId, fieldId, name, transform);
return Objects.hashCode(sourceIds, fieldId, name, transform);
}
}
108 changes: 65 additions & 43 deletions api/src/main/java/org/apache/iceberg/PartitionSpec.java
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ public UnboundPartitionSpec toUnbound() {

for (PartitionField field : fields) {
builder.addField(
field.transform().toString(), field.sourceId(), field.fieldId(), field.name());
field.transform().toString(), field.sourceIds(), field.fieldId(), field.name());
}

return builder.build();
Expand Down Expand Up @@ -198,7 +198,7 @@ public Class<?>[] javaClasses() {
}

private Type resultType(PartitionField field) {
Type sourceType = schema.findType(field.sourceId());
Type sourceType = schema.findType(field.sourceIds().get(0));
if (sourceType == null) {
// When the source field has been dropped, the source type has been lost
// Transforms with a fixed result type still work, so use unknown for source
Expand Down Expand Up @@ -254,7 +254,7 @@ public boolean compatibleWith(PartitionSpec other) {
for (int i = 0; i < fields.length; i += 1) {
PartitionField thisField = fields[i];
PartitionField thatField = other.fields[i];
if (thisField.sourceId() != thatField.sourceId()
if (!thisField.sourceIds().equals(thatField.sourceIds())
|| !thisField.transform().toString().equals(thatField.transform().toString())
|| !thisField.name().equals(thatField.name())) {
return false;
Expand Down Expand Up @@ -303,7 +303,9 @@ private ListMultimap<Integer, PartitionField> lazyFieldsBySourceId() {
Multimaps.newListMultimap(
Maps.newHashMap(), () -> Lists.newArrayListWithCapacity(fields.length));
for (PartitionField field : fields) {
multiMap.put(field.sourceId(), field);
for (int sourceId : field.sourceIds()) {
multiMap.put(sourceId, field);
}
}
this.fieldsBySourceId = multiMap;
}
Expand All @@ -322,7 +324,7 @@ public Set<Integer> identitySourceIds() {
Set<Integer> sourceIds = Sets.newHashSet();
for (PartitionField field : fields()) {
if (field.transform().isIdentity()) {
sourceIds.add(field.sourceId());
sourceIds.addAll(field.sourceIds());
}
}

Expand Down Expand Up @@ -433,15 +435,17 @@ private void checkAndAddPartitionName(String name, Integer sourceColumnId) {
}

private void checkForRedundantPartitions(PartitionField field) {
Map.Entry<Integer, String> dedupKey =
new AbstractMap.SimpleEntry<>(field.sourceId(), field.transform().dedupName());
PartitionField partitionField = dedupFields.get(dedupKey);
Preconditions.checkArgument(
partitionField == null,
"Cannot add redundant partition: %s conflicts with %s",
partitionField,
field);
dedupFields.put(dedupKey, field);
for (int sourceId : field.sourceIds()) {
Map.Entry<Integer, String> dedupKey =
new AbstractMap.SimpleEntry<>(sourceId, field.transform().dedupName());
PartitionField partitionField = dedupFields.get(dedupKey);
Preconditions.checkArgument(
partitionField == null,
"Cannot add redundant partition: %s conflicts with %s",
partitionField,
field);
dedupFields.put(dedupKey, field);
}
}

public Builder caseSensitive(boolean sensitive) {
Expand Down Expand Up @@ -627,6 +631,22 @@ Builder add(int sourceId, int fieldId, String name, Transform<?, ?> transform) {
return this;
}

Builder add(List<Integer> sourceIds, int fieldId, String name, Transform<?, ?> transform) {
if (sourceIds.size() == 1) {
checkAndAddPartitionName(name, sourceIds.get(0));
} else {
checkAndAddPartitionName(name);
}

fields.add(new PartitionField(sourceIds, fieldId, name, transform));
lastAssignedFieldId.getAndAccumulate(fieldId, Math::max);
return this;
}

Builder add(List<Integer> sourceIds, String name, Transform<?, ?> transform) {
return add(sourceIds, nextFieldId(), name, transform);
}

public PartitionSpec build() {
return build(false);
}
Expand All @@ -649,37 +669,39 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) {
static void checkCompatibility(PartitionSpec spec, Schema schema, boolean allowMissingFields) {
final Map<Integer, Integer> parents = TypeUtil.indexParents(schema.asStruct());
for (PartitionField field : spec.fields) {
Type sourceType = schema.findType(field.sourceId());
Transform<?, ?> transform = field.transform();
// In the case the underlying field is dropped, we cannot check if they are compatible
if (allowMissingFields && sourceType == null) {
continue;
}
// In the case of a Version 1 partition-spec field gets deleted,
// it is replaced with a void transform, see:
// https://iceberg.apache.org/spec/#partition-transforms
// We don't care about the source type since a VoidTransform is always compatible and skip the
// checks
if (!transform.equals(Transforms.alwaysNull())) {
ValidationException.check(
sourceType != null, "Cannot find source column for partition field: %s", field);
ValidationException.check(
sourceType.isPrimitiveType(),
"Cannot partition by non-primitive source field: %s",
sourceType);
ValidationException.check(
transform.canTransform(sourceType),
"Invalid source type %s for transform: %s",
sourceType,
transform);
// The only valid parent types for a PartitionField are StructTypes. This must be checked
// recursively.
Integer parentId = parents.get(field.sourceId());
while (parentId != null) {
Type parentType = schema.findType(parentId);
for (int srcId : field.sourceIds()) {
Type sourceType = schema.findType(srcId);
// In the case the underlying field is dropped, we cannot check if they are compatible
if (allowMissingFields && sourceType == null) {
continue;
}
// In the case of a Version 1 partition-spec field gets deleted,
// it is replaced with a void transform, see:
// https://iceberg.apache.org/spec/#partition-transforms
// We don't care about the source type since a VoidTransform is always compatible and skip
// the checks
if (!transform.equals(Transforms.alwaysNull())) {
ValidationException.check(
sourceType != null, "Cannot find source column for partition field: %s", field);
ValidationException.check(
parentType.isStructType(), "Invalid partition field parent: %s", parentType);
parentId = parents.get(parentId);
sourceType.isPrimitiveType(),
"Cannot partition by non-primitive source field: %s",
sourceType);
ValidationException.check(
transform.canTransform(sourceType),
"Invalid source type %s for transform: %s",
sourceType,
transform);
// The only valid parent types for a PartitionField are StructTypes. This must be checked
// recursively.
Integer parentId = parents.get(srcId);
while (parentId != null) {
Type parentType = schema.findType(parentId);
ValidationException.check(
parentType.isStructType(), "Invalid partition field parent: %s", parentType);
parentId = parents.get(parentId);
}
}
}
}
Expand Down
42 changes: 35 additions & 7 deletions api/src/main/java/org/apache/iceberg/SortField.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,31 @@
package org.apache.iceberg;

import java.io.Serializable;
import java.util.List;
import java.util.Objects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.transforms.Transform;

/** A field in a {@link SortOrder}. */
public class SortField implements Serializable {

private final Transform<?, ?> transform;
private final int sourceId;
private final List<Integer> sourceIds;
private final SortDirection direction;
private final NullOrder nullOrder;

SortField(Transform<?, ?> transform, int sourceId, SortDirection direction, NullOrder nullOrder) {
this(transform, ImmutableList.of(sourceId), direction, nullOrder);
}

SortField(
Transform<?, ?> transform,
List<Integer> sourceIds,
SortDirection direction,
NullOrder nullOrder) {
this.transform = transform;
this.sourceId = sourceId;
this.sourceIds = ImmutableList.copyOf(sourceIds);
this.direction = direction;
this.nullOrder = nullOrder;
}
Expand All @@ -51,7 +62,21 @@ public <S, T> Transform<S, T> transform() {

/** Returns the field id of the source field in the {@link SortOrder sort order's} table schema */
public int sourceId() {
return sourceId;
Preconditions.checkState(
sourceIds.size() == 1,
"Use sourceIds() for multi-argument transforms, found %s source ids",
sourceIds.size());
return sourceIds.get(0);
}

/**
* Returns the field ids of all source fields for this sort field.
*
* <p>For single-argument transforms, this list contains one element. For multi-argument
* transforms, this list contains multiple source field ids.
*/
public List<Integer> sourceIds() {
return sourceIds;
}

/** Returns the sort direction */
Expand All @@ -73,7 +98,7 @@ public NullOrder nullOrder() {
public boolean satisfies(SortField other) {
if (Objects.equals(this, other)) {
return true;
} else if (sourceId != other.sourceId
} else if (!sourceIds.equals(other.sourceIds)
|| direction != other.direction
|| nullOrder != other.nullOrder) {
return false;
Expand All @@ -84,7 +109,10 @@ public boolean satisfies(SortField other) {

@Override
public String toString() {
return transform + "(" + sourceId + ") " + direction + " " + nullOrder;
if (sourceIds.size() == 1) {
return transform + "(" + sourceIds.get(0) + ") " + direction + " " + nullOrder;
}
return transform + "(" + sourceIds + ") " + direction + " " + nullOrder;
}

@Override
Expand All @@ -97,13 +125,13 @@ public boolean equals(Object other) {

SortField that = (SortField) other;
return transform.toString().equals(that.transform.toString())
&& sourceId == that.sourceId
&& sourceIds.equals(that.sourceIds)
&& direction == that.direction
&& nullOrder == that.nullOrder;
}

@Override
public int hashCode() {
return Objects.hash(transform, sourceId, direction, nullOrder);
return Objects.hash(transform, sourceIds, direction, nullOrder);
}
}
Loading
Loading