upperBounds();
+ /**
+ * Returns if collected, map from column ID to its average value size in memory (uncompressed) in
+ * bytes over non-null values, null otherwise.
+ *
+ * This statistic is not persisted in manifests prior to v4, so it is generally only present on
+ * newly created or written files, and is null for files read back from such manifests.
+ */
+ default Map avgValueSizes() {
+ return null;
+ }
+
/**
* Returns metadata about how this file is encrypted, or null if the file is stored in plain text.
*/
@@ -187,7 +198,7 @@ default Long firstRowId() {
* to copy data without stats when collecting files.
*
* @return a copy of this data file, without lower bounds, upper bounds, value counts, null value
- * counts, or nan value counts
+ * counts, nan value counts, or average value sizes
*/
F copyWithoutStats();
@@ -198,7 +209,7 @@ default Long firstRowId() {
*
* @param requestedColumnIds column IDs for which to keep stats.
* @return a copy of data file, with lower bounds, upper bounds, value counts, null value counts,
- * and nan value counts for only specific columns.
+ * nan value counts, and average value sizes for only specific columns.
*/
default F copyWithStats(Set requestedColumnIds) {
throw new UnsupportedOperationException(
@@ -211,8 +222,8 @@ default F copyWithStats(Set requestedColumnIds) {
*
* @param withStats Will copy this file without file stats if set to false.
* @return a copy of this data file. If withStats is set to false the
- * file will not contain lower bounds, upper bounds, value counts, null value counts, or nan
- * value counts
+ * file will not contain lower bounds, upper bounds, value counts, null value counts, nan
+ * value counts, or average value sizes
*/
default F copy(boolean withStats) {
return withStats ? copy() : copyWithoutStats();
diff --git a/api/src/main/java/org/apache/iceberg/DataFile.java b/api/src/main/java/org/apache/iceberg/DataFile.java
index 64338228349a..b6a72b5b339d 100644
--- a/api/src/main/java/org/apache/iceberg/DataFile.java
+++ b/api/src/main/java/org/apache/iceberg/DataFile.java
@@ -164,4 +164,14 @@ default FileContent content() {
default List equalityFieldIds() {
return null;
}
+
+ /**
+ * Returns the deletion vector co-located or tracked with this data file, or null if there isn't
+ * one.
+ *
+ * A delete manifest can contain a deletion vector even if this co-located DV is null.
+ */
+ default DeletionVector deletionVector() {
+ return null;
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/DeletionVector.java b/api/src/main/java/org/apache/iceberg/DeletionVector.java
similarity index 85%
rename from core/src/main/java/org/apache/iceberg/DeletionVector.java
rename to api/src/main/java/org/apache/iceberg/DeletionVector.java
index 0fc8f259f075..494bbedd7951 100644
--- a/core/src/main/java/org/apache/iceberg/DeletionVector.java
+++ b/api/src/main/java/org/apache/iceberg/DeletionVector.java
@@ -18,6 +18,7 @@
*/
package org.apache.iceberg;
+import java.nio.ByteBuffer;
import org.apache.iceberg.types.Types;
/**
@@ -26,7 +27,7 @@
*
Tracks where a DV blob can be read. The DV blob follows the format defined by the
* deletion-vector-v1 blob type in the Puffin spec.
*/
-interface DeletionVector {
+public interface DeletionVector {
Types.NestedField LOCATION =
Types.NestedField.required(
155, "location", Types.StringType.get(), "Location of the file containing the DV");
@@ -45,9 +46,15 @@ interface DeletionVector {
"cardinality",
Types.LongType.get(),
"Number of set bits (deleted rows) in the vector");
+ Types.NestedField KEY_METADATA =
+ Types.NestedField.optional(
+ 149,
+ "key_metadata",
+ Types.BinaryType.get(),
+ "Implementation-specific key metadata for encryption");
static Types.StructType schema() {
- return Types.StructType.of(LOCATION, OFFSET, SIZE_IN_BYTES, CARDINALITY);
+ return Types.StructType.of(LOCATION, OFFSET, SIZE_IN_BYTES, CARDINALITY, KEY_METADATA);
}
/** Returns the location of the file containing the deletion vector. */
@@ -62,6 +69,9 @@ static Types.StructType schema() {
/** Returns the number of set bits (deleted rows) in the vector. */
long cardinality();
+ /** Returns encryption key metadata, or null if the file is not encrypted. */
+ ByteBuffer keyMetadata();
+
/** Copies this deletion vector. */
DeletionVector copy();
}
diff --git a/api/src/main/java/org/apache/iceberg/DistributionMode.java b/api/src/main/java/org/apache/iceberg/DistributionMode.java
index a7625675e1d7..27c04b5af027 100644
--- a/api/src/main/java/org/apache/iceberg/DistributionMode.java
+++ b/api/src/main/java/org/apache/iceberg/DistributionMode.java
@@ -54,7 +54,7 @@ public String modeName() {
public static DistributionMode fromName(String modeName) {
Preconditions.checkArgument(null != modeName, "Invalid distribution mode: null");
try {
- return DistributionMode.valueOf(modeName.toUpperCase(Locale.ENGLISH));
+ return DistributionMode.valueOf(modeName.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(String.format("Invalid distribution mode: %s", modeName));
}
diff --git a/api/src/main/java/org/apache/iceberg/FileFormat.java b/api/src/main/java/org/apache/iceberg/FileFormat.java
index 91d1e3c7a842..b09e88be2961 100644
--- a/api/src/main/java/org/apache/iceberg/FileFormat.java
+++ b/api/src/main/java/org/apache/iceberg/FileFormat.java
@@ -79,7 +79,7 @@ public static FileFormat fromFileName(CharSequence filename) {
public static FileFormat fromString(String fileFormat) {
Preconditions.checkArgument(null != fileFormat, "Invalid file format: null");
try {
- return FileFormat.valueOf(fileFormat.toUpperCase(Locale.ENGLISH));
+ return FileFormat.valueOf(fileFormat.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(String.format("Invalid file format: %s", fileFormat), e);
}
diff --git a/api/src/main/java/org/apache/iceberg/Files.java b/api/src/main/java/org/apache/iceberg/Files.java
index 16d3b663ad35..aa3fc8e35917 100644
--- a/api/src/main/java/org/apache/iceberg/Files.java
+++ b/api/src/main/java/org/apache/iceberg/Files.java
@@ -40,6 +40,9 @@ public static OutputFile localOutput(File file) {
}
public static OutputFile localOutput(String file) {
+ if (file.startsWith("file:")) {
+ return localOutput(new File(file.replaceFirst("file:", "")));
+ }
return localOutput(Paths.get(file).toAbsolutePath().toFile());
}
@@ -70,10 +73,8 @@ public PositionOutputStream create() {
@Override
public PositionOutputStream createOrOverwrite() {
- if (file.exists()) {
- if (!file.delete()) {
- throw new RuntimeIOException("Failed to delete: %s", file);
- }
+ if (file.exists() && !file.delete()) {
+ throw new RuntimeIOException("Failed to delete: %s", file);
}
return create();
}
diff --git a/api/src/main/java/org/apache/iceberg/ManageSnapshots.java b/api/src/main/java/org/apache/iceberg/ManageSnapshots.java
index 36e98075656f..5774c0d7f5ac 100644
--- a/api/src/main/java/org/apache/iceberg/ManageSnapshots.java
+++ b/api/src/main/java/org/apache/iceberg/ManageSnapshots.java
@@ -66,6 +66,7 @@ public interface ManageSnapshots extends PendingUpdate {
*
* @param snapshotId long id of snapshot id to roll back table to. Must be an ancestor of the
* current snapshot
+ * @return this for method chaining
* @throws IllegalArgumentException If the table has no snapshot with the given id
* @throws ValidationException If given snapshot id is not an ancestor of the current state
*/
@@ -110,7 +111,7 @@ default ManageSnapshots createBranch(String name) {
* Create a new tag pointing to the given snapshot id
*
* @param name tag name
- * @param snapshotId snapshotId for the head of the new branch.
+ * @param snapshotId snapshotId for the head of the new tag.
* @return this for method chaining
* @throws IllegalArgumentException if a tag with the given name already exists
*/
@@ -130,6 +131,7 @@ default ManageSnapshots createBranch(String name) {
*
* @param name name of branch to rename
* @param newName the desired new name of the branch
+ * @return this for method chaining
* @throws IllegalArgumentException if the branch to rename does not exist or if there is already
* a branch with the same name as the desired new name.
*/
@@ -140,7 +142,7 @@ default ManageSnapshots createBranch(String name) {
*
* @param name tag name
* @return this for method chaining
- * @throws IllegalArgumentException if the branch does not exist
+ * @throws IllegalArgumentException if the tag does not exist
*/
ManageSnapshots removeTag(String name);
diff --git a/api/src/main/java/org/apache/iceberg/ManifestBitmap.java b/api/src/main/java/org/apache/iceberg/ManifestBitmap.java
new file mode 100644
index 000000000000..a6a41af41479
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/ManifestBitmap.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+
+/** A bitmap that is serialized inline in a metadata file. */
+public interface ManifestBitmap {
+ /** Number of bits set in this bitmap. */
+ int cardinality();
+
+ /** Return whether the bit at {@code position} is set. */
+ boolean isSet(int position);
+
+ /** Return the serialized bitmap as a {@link ByteBuffer}. */
+ ByteBuffer buffer();
+}
diff --git a/api/src/main/java/org/apache/iceberg/ManifestFile.java b/api/src/main/java/org/apache/iceberg/ManifestFile.java
index 2f732aef427f..cfedb78a7cee 100644
--- a/api/src/main/java/org/apache/iceberg/ManifestFile.java
+++ b/api/src/main/java/org/apache/iceberg/ManifestFile.java
@@ -126,7 +126,7 @@ static Schema schema() {
/** Returns length of the manifest file. */
long length();
- /** Returns iD of the {@link PartitionSpec} used to write the manifest file. */
+ /** Returns ID of the {@link PartitionSpec} used to write the manifest file. */
int partitionSpecId();
/** Returns the content stored in the manifest; either DATA or DELETES. */
@@ -138,7 +138,7 @@ static Schema schema() {
/** Returns the lowest data sequence number of any live file in the manifest. */
long minSequenceNumber();
- /** Returns iD of the snapshot that added the manifest file to table metadata. */
+ /** Returns ID of the snapshot that added the manifest file to table metadata. */
Long snapshotId();
/**
@@ -210,6 +210,16 @@ default Long firstRowId() {
return null;
}
+ /** Returns the manifest deletion vector, or null if absent. */
+ default ManifestBitmap manifestDeletionVector() {
+ return null;
+ }
+
+ /** Returns the writer format version of the manifest, or 0 if it predates format tracking. */
+ default int formatVersion() {
+ return 0;
+ }
+
/**
* Copies this {@link ManifestFile manifest file}. Readers can reuse manifest file instances; use
* this method to make defensive copies.
diff --git a/api/src/main/java/org/apache/iceberg/Metrics.java b/api/src/main/java/org/apache/iceberg/Metrics.java
index eb13caf5e80a..ce16c68acb32 100644
--- a/api/src/main/java/org/apache/iceberg/Metrics.java
+++ b/api/src/main/java/org/apache/iceberg/Metrics.java
@@ -38,15 +38,32 @@ public class Metrics implements Serializable {
private Map nanValueCounts = null;
private Map lowerBounds = null;
private Map upperBounds = null;
+ private Map avgValueSizes = null;
// this is not serialized with all the other fields
private Map originalTypes = null;
+ /** Creates an empty metrics instance with no statistics set. */
public Metrics() {}
+ /**
+ * Creates a new metrics instance with only the row count set.
+ *
+ * @param rowCount the number of rows (records) in the file
+ */
public Metrics(long rowCount) {
this.rowCount = rowCount;
}
+ /**
+ * Creates a new metrics instance.
+ *
+ * @param rowCount the number of rows (records) in the file, or null if unknown
+ * @param columnSizes a map of field id to the size in bytes of the column, or null if unknown
+ * @param valueCounts a map of field id to the number of all values (including nulls, NaN, and
+ * repeated), or null if unknown
+ * @param nullValueCounts a map of field id to the number of null values, or null if unknown
+ * @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
+ */
public Metrics(
Long rowCount,
Map columnSizes,
@@ -56,6 +73,18 @@ public Metrics(
this(rowCount, columnSizes, valueCounts, nullValueCounts, nanValueCounts, null, null, null);
}
+ /**
+ * Creates a new metrics instance.
+ *
+ * @param rowCount the number of rows (records) in the file, or null if unknown
+ * @param columnSizes a map of field id to the size in bytes of the column, or null if unknown
+ * @param valueCounts a map of field id to the number of all values (including nulls, NaN, and
+ * repeated), or null if unknown
+ * @param nullValueCounts a map of field id to the number of null values, or null if unknown
+ * @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
+ * @param lowerBounds a map of field id to the lower bound of the column, or null if unknown
+ * @param upperBounds a map of field id to the upper bound of the column, or null if unknown
+ */
public Metrics(
Long rowCount,
Map columnSizes,
@@ -75,6 +104,20 @@ public Metrics(
null);
}
+ /**
+ * Creates a new metrics instance.
+ *
+ * @param rowCount the number of rows (records) in the file, or null if unknown
+ * @param columnSizes a map of field id to the size in bytes of the column, or null if unknown
+ * @param valueCounts a map of field id to the number of all values (including nulls, NaN, and
+ * repeated), or null if unknown
+ * @param nullValueCounts a map of field id to the number of null values, or null if unknown
+ * @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
+ * @param lowerBounds a map of field id to the lower bound of the column, or null if unknown
+ * @param upperBounds a map of field id to the upper bound of the column, or null if unknown
+ * @param originalTypes a map of field id to the original type of the lower/upper bound, or null
+ * if unknown
+ */
public Metrics(
Long rowCount,
Map columnSizes,
@@ -84,6 +127,44 @@ public Metrics(
Map lowerBounds,
Map upperBounds,
Map originalTypes) {
+ this(
+ rowCount,
+ columnSizes,
+ valueCounts,
+ nullValueCounts,
+ nanValueCounts,
+ lowerBounds,
+ upperBounds,
+ null /* avgValueSizes */,
+ originalTypes);
+ }
+
+ /**
+ * Creates a new metrics instance.
+ *
+ * @param rowCount the number of rows (records) in the file, or null if unknown
+ * @param columnSizes a map of field id to the size in bytes of the column, or null if unknown
+ * @param valueCounts a map of field id to the number of all values (including nulls, NaN, and
+ * repeated), or null if unknown
+ * @param nullValueCounts a map of field id to the number of null values, or null if unknown
+ * @param nanValueCounts a map of field id to the number of NaN values, or null if unknown
+ * @param lowerBounds a map of field id to the lower bound of the column, or null if unknown
+ * @param upperBounds a map of field id to the upper bound of the column, or null if unknown
+ * @param avgValueSizes a map of field id to the average size in bytes of the column's non-null
+ * values, or null if unknown
+ * @param originalTypes a map of field id to the original type of the lower/upper bound, or null
+ * if unknown
+ */
+ public Metrics(
+ Long rowCount,
+ Map columnSizes,
+ Map valueCounts,
+ Map nullValueCounts,
+ Map nanValueCounts,
+ Map lowerBounds,
+ Map upperBounds,
+ Map avgValueSizes,
+ Map originalTypes) {
this.rowCount = rowCount;
this.columnSizes = columnSizes;
this.valueCounts = valueCounts;
@@ -91,6 +172,7 @@ public Metrics(
this.nanValueCounts = nanValueCounts;
this.lowerBounds = lowerBounds;
this.upperBounds = upperBounds;
+ this.avgValueSizes = avgValueSizes;
this.originalTypes = originalTypes;
}
@@ -162,6 +244,16 @@ public Map upperBounds() {
return upperBounds;
}
+ /**
+ * Get the average value size in memory (uncompressed) in bytes over non-null values, for all
+ * fields where it was collected.
+ *
+ * @return a Map of fieldId to average value size in bytes
+ */
+ public Map avgValueSizes() {
+ return avgValueSizes;
+ }
+
/**
* Get the non-null original types for the upper/lower bound for all fields in a file.
*
@@ -186,6 +278,7 @@ private void writeObject(ObjectOutputStream out) throws IOException {
writeByteBufferMap(out, lowerBounds);
writeByteBufferMap(out, upperBounds);
+ out.writeObject(avgValueSizes);
}
private static void writeByteBufferMap(
@@ -222,6 +315,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE
lowerBounds = readByteBufferMap(in);
upperBounds = readByteBufferMap(in);
+ avgValueSizes = (Map) in.readObject();
}
@SuppressWarnings("DangerousJavaDeserialization")
diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
index 90d2dc259dd1..9c1528582c8a 100644
--- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
@@ -49,6 +49,15 @@
*
* Partition data is produced by transforming columns in a table. Each column transform is
* represented by a named {@link PartitionField}.
+ *
+ *
Partition specs are created using a builder obtained from {@link #builderFor(Schema)}:
+ *
+ *
{@code
+ * PartitionSpec spec = PartitionSpec.builderFor(schema)
+ * .hour("ts")
+ * .bucket("id", 10)
+ * .build();
+ * }
*/
public class PartitionSpec implements Serializable {
// IDs for partition fields start at 1000
@@ -130,15 +139,8 @@ public StructType partitionType() {
List structFields = Lists.newArrayListWithExpectedSize(fields.length);
for (PartitionField field : fields) {
- Type sourceType = schema.findType(field.sourceId());
- Type resultType = field.transform().getResultType(sourceType);
-
- // When the source field has been dropped we cannot determine the type
- if (sourceType == null) {
- resultType = Types.UnknownType.get();
- }
-
- structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType));
+ structFields.add(
+ Types.NestedField.optional(field.fieldId(), field.name(), resultType(field)));
}
this.lazyPartitionType = Types.StructType.of(structFields);
@@ -183,14 +185,7 @@ public Class>[] javaClasses() {
if (field.transform() instanceof UnknownTransform) {
classes[i] = Object.class;
} else {
- Type sourceType = schema.findType(field.sourceId());
- if (null == sourceType) {
- // When the source field has been dropped we cannot determine the type
- sourceType = Types.UnknownType.get();
- }
-
- Type result = field.transform().getResultType(sourceType);
- classes[i] = result.typeId().javaClass();
+ classes[i] = resultType(field).typeId().javaClass();
}
}
@@ -202,6 +197,17 @@ public Class>[] javaClasses() {
return lazyJavaClasses;
}
+ private Type resultType(PartitionField field) {
+ Type sourceType = schema.findType(field.sourceId());
+ if (sourceType == null) {
+ // When the source field has been dropped, the source type has been lost
+ // Transforms with a fixed result type still work, so use unknown for source
+ sourceType = Types.UnknownType.get();
+ }
+
+ return field.transform().getResultType(sourceType);
+ }
+
@SuppressWarnings("unchecked")
private T get(StructLike data, int pos, Class> javaClass) {
return data.get(pos, (Class) javaClass);
diff --git a/api/src/main/java/org/apache/iceberg/PartitionStatisticsScan.java b/api/src/main/java/org/apache/iceberg/PartitionStatisticsScan.java
index 18d8b2031821..9b72b45f23de 100644
--- a/api/src/main/java/org/apache/iceberg/PartitionStatisticsScan.java
+++ b/api/src/main/java/org/apache/iceberg/PartitionStatisticsScan.java
@@ -42,6 +42,17 @@ public interface PartitionStatisticsScan {
*/
PartitionStatisticsScan filter(Expression filter);
+ /**
+ * Create a new scan from this that will configure whether column name matches in filter
+ * expressions are case-sensitive. Default is true.
+ *
+ * @param caseSensitive whether column name matches in filter expressions should be case-sensitive
+ * @return a new scan based on this with the given case sensitivity
+ */
+ default PartitionStatisticsScan caseSensitive(boolean caseSensitive) {
+ throw new UnsupportedOperationException("caseSensitive is not supported");
+ }
+
/**
* Create a new scan from this with the schema as its projection.
*
diff --git a/api/src/main/java/org/apache/iceberg/RewriteJobOrder.java b/api/src/main/java/org/apache/iceberg/RewriteJobOrder.java
index 7038f127afaa..85867c319946 100644
--- a/api/src/main/java/org/apache/iceberg/RewriteJobOrder.java
+++ b/api/src/main/java/org/apache/iceberg/RewriteJobOrder.java
@@ -59,7 +59,7 @@ public static RewriteJobOrder fromName(String orderName) {
// Replace the hyphen in order name with underscore to map to the enum value. For example:
// bytes-asc to BYTES_ASC
try {
- return RewriteJobOrder.valueOf(orderName.replaceFirst("-", "_").toUpperCase(Locale.ENGLISH));
+ return RewriteJobOrder.valueOf(orderName.replaceFirst("-", "_").toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format("Invalid rewrite job order name: %s", orderName), e);
diff --git a/api/src/main/java/org/apache/iceberg/SnapshotRefType.java b/api/src/main/java/org/apache/iceberg/SnapshotRefType.java
index 361d40e7aac1..ce9b3449c4ea 100644
--- a/api/src/main/java/org/apache/iceberg/SnapshotRefType.java
+++ b/api/src/main/java/org/apache/iceberg/SnapshotRefType.java
@@ -28,7 +28,7 @@ public enum SnapshotRefType {
public static SnapshotRefType fromString(String snapshotRefType) {
Preconditions.checkArgument(null != snapshotRefType, "Invalid snapshot ref type: null");
try {
- return SnapshotRefType.valueOf(snapshotRefType.toUpperCase(Locale.ENGLISH));
+ return SnapshotRefType.valueOf(snapshotRefType.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format("Invalid snapshot ref type: %s", snapshotRefType), e);
diff --git a/api/src/main/java/org/apache/iceberg/SnapshotUpdate.java b/api/src/main/java/org/apache/iceberg/SnapshotUpdate.java
index 73509c15384f..9ce946c0a055 100644
--- a/api/src/main/java/org/apache/iceberg/SnapshotUpdate.java
+++ b/api/src/main/java/org/apache/iceberg/SnapshotUpdate.java
@@ -60,10 +60,27 @@ public interface SnapshotUpdate extends PendingUpdate {
*/
ThisT scanManifestsWith(ExecutorService executorService);
+ /**
+ * Use a particular executor to write manifests during commit with the specified parallelism. The
+ * default worker pool will be used by default.
+ *
+ * The parallelism parameter controls how many manifest writers are used, which informs the
+ * number of manifest files produced. The executor provides the threads for parallel execution.
+ *
+ * @param executorService the provided executor
+ * @param parallelism the number of parallel manifest writers to use
+ * @return this for method chaining
+ */
+ default ThisT writeManifestsWith(ExecutorService executorService, int parallelism) {
+ throw new UnsupportedOperationException(
+ this.getClass().getName() + " does not support writeManifestsWith");
+ }
+
/**
* Perform operations on a particular branch
*
* @param branch which is name of SnapshotRef of type branch.
+ * @return this for method chaining
*/
default ThisT toBranch(String branch) {
throw new UnsupportedOperationException(
diff --git a/api/src/main/java/org/apache/iceberg/SortDirection.java b/api/src/main/java/org/apache/iceberg/SortDirection.java
index d76fe199f918..b92ea4597c8a 100644
--- a/api/src/main/java/org/apache/iceberg/SortDirection.java
+++ b/api/src/main/java/org/apache/iceberg/SortDirection.java
@@ -28,7 +28,7 @@ public enum SortDirection {
public static SortDirection fromString(String directionAsString) {
Preconditions.checkArgument(null != directionAsString, "Invalid sort direction: null");
try {
- return SortDirection.valueOf(directionAsString.toUpperCase(Locale.ENGLISH));
+ return SortDirection.valueOf(directionAsString.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format("Invalid sort direction: %s", directionAsString), e);
diff --git a/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java b/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
index a4994d22001d..d97c7112880d 100644
--- a/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
@@ -47,8 +47,8 @@ public interface UpdatePartitionSpec extends PendingUpdate {
*
* @param sourceName source column name in the table schema
* @return this for method chaining
- * @throws IllegalArgumentException If the an identity partition field for the source already
- * exists, or if this change conflicts with other additions, removals, or renames.
+ * @throws IllegalArgumentException If an identity partition field for the source already exists,
+ * or if this change conflicts with other additions, removals, or renames.
*/
UpdatePartitionSpec addField(String sourceName);
@@ -65,7 +65,7 @@ public interface UpdatePartitionSpec extends PendingUpdate {
*
* @param term source column name in the table schema
* @return this for method chaining
- * @throws IllegalArgumentException If the a partition field for the transform and source already
+ * @throws IllegalArgumentException If a partition field for the transform and source already
* exists, or if this change conflicts with other additions, removals, or renames.
*/
UpdatePartitionSpec addField(Term term);
@@ -83,7 +83,7 @@ public interface UpdatePartitionSpec extends PendingUpdate {
* @param name name for the partition field
* @param term expression for the partition transform
* @return this for method chaining
- * @throws IllegalArgumentException If the a partition field for the transform and source already
+ * @throws IllegalArgumentException If a partition field for the transform and source already
* exists, if a partition field with the given name already exists, or if this change
* conflicts with other additions, removals, or renames.
*/
@@ -94,8 +94,8 @@ public interface UpdatePartitionSpec extends PendingUpdate {
*
* @param name name of the partition field to remove
* @return this for method chaining
- * @throws IllegalArgumentException If the a partition field with the given name does not exist,
- * or if this change conflicts with other additions, removals, or renames.
+ * @throws IllegalArgumentException If a partition field with the given name does not exist, or if
+ * this change conflicts with other additions, removals, or renames.
*/
UpdatePartitionSpec removeField(String name);
@@ -107,8 +107,8 @@ public interface UpdatePartitionSpec extends PendingUpdate {
*
* @param term expression for the partition transform to remove
* @return this for method chaining
- * @throws IllegalArgumentException If the a partition field with the given transform and source
- * does not exist, or if this change conflicts with other additions, removals, or renames.
+ * @throws IllegalArgumentException If a partition field with the given transform and source does
+ * not exist, or if this change conflicts with other additions, removals, or renames.
*/
UpdatePartitionSpec removeField(Term term);
diff --git a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
index 240e34113721..3bb18d8989af 100644
--- a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
+++ b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
@@ -94,4 +94,10 @@ default RemoveDanglingDeleteFiles removeDanglingDeleteFiles(Table table) {
throw new UnsupportedOperationException(
this.getClass().getName() + " does not implement removeDanglingDeleteFiles");
}
+
+ /** Instantiates an action to repair a table. */
+ default RepairTable repairTable(Table table) {
+ throw new UnsupportedOperationException(
+ this.getClass().getName() + " does not implement repairTable");
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
index ab12a3b7c1e3..c5766f981db9 100644
--- a/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
+++ b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
@@ -163,7 +163,7 @@ enum PrefixMismatchMode {
public static PrefixMismatchMode fromString(String modeAsString) {
Preconditions.checkArgument(modeAsString != null, "Invalid mode: null");
try {
- return PrefixMismatchMode.valueOf(modeAsString.toUpperCase(Locale.ENGLISH));
+ return PrefixMismatchMode.valueOf(modeAsString.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(String.format("Invalid mode: %s", modeAsString), e);
}
diff --git a/api/src/main/java/org/apache/iceberg/actions/MigrateTable.java b/api/src/main/java/org/apache/iceberg/actions/MigrateTable.java
index 8f9e8d69c90a..7a82ddabb104 100644
--- a/api/src/main/java/org/apache/iceberg/actions/MigrateTable.java
+++ b/api/src/main/java/org/apache/iceberg/actions/MigrateTable.java
@@ -72,6 +72,18 @@ default MigrateTable executeWith(ExecutorService service) {
throw new UnsupportedOperationException("Setting executor service is not supported");
}
+ /**
+ * Enables ignoring {@link java.io.FileNotFoundException} when listing source data files. When
+ * enabled, source data files that have disappeared (for example, because a partition directory
+ * was removed by concurrent cleanup) are skipped with a warning instead of failing the migration.
+ * The default is to fail.
+ *
+ * @return this for method chaining
+ */
+ default MigrateTable ignoreMissingFiles() {
+ throw new UnsupportedOperationException("Ignoring missing files is not supported");
+ }
+
/** The action result that contains a summary of the execution. */
interface Result {
/** Returns the number of migrated data files. */
diff --git a/api/src/main/java/org/apache/iceberg/actions/RepairTable.java b/api/src/main/java/org/apache/iceberg/actions/RepairTable.java
new file mode 100644
index 000000000000..7df92f6f3f3d
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/actions/RepairTable.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.actions;
+
+import org.apache.iceberg.ManifestFile;
+
+/**
+ * An action that repairs a table.
+ *
+ * The repairs to perform are selected through the configuration methods of this action.
+ * Implementations rewrite the affected metadata, producing a new set of manifests in which the
+ * repaired entries replace the corrupt ones.
+ */
+public interface RepairTable extends SnapshotUpdate {
+
+ /**
+ * Repairs incorrect metrics of manifest entries, such as record counts, file sizes and column
+ * level statistics, by comparing them against the underlying data and delete files.
+ *
+ * @return this for method chaining
+ */
+ RepairTable repairFileMetrics();
+
+ /**
+ * Determines the repairs that would be performed without actually committing the operation to the
+ * table.
+ *
+ * @return this for method chaining
+ */
+ RepairTable dryRun();
+
+ /** The action result that contains a summary of the execution. */
+ interface Result {
+ /** Returns the repaired manifests. */
+ Iterable repairedManifests();
+
+ /** Returns the number of manifest entries that were repaired. */
+ long repairedEntryCount();
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/actions/RewriteDataFiles.java b/api/src/main/java/org/apache/iceberg/actions/RewriteDataFiles.java
index 0459538d1cda..0db2e6deb9ce 100644
--- a/api/src/main/java/org/apache/iceberg/actions/RewriteDataFiles.java
+++ b/api/src/main/java/org/apache/iceberg/actions/RewriteDataFiles.java
@@ -188,6 +188,17 @@ default RewriteDataFiles zOrder(String... columns) {
"Z-ORDER Rewrite Strategy not implemented for this framework");
}
+ /**
+ * Choose HILBERT as a strategy for this rewrite operation with a specified list of columns to use
+ *
+ * @param columns Columns to be used to generate Hilbert values
+ * @return this for method chaining
+ */
+ default RewriteDataFiles hilbert(String... columns) {
+ throw new UnsupportedOperationException(
+ "HILBERT Rewrite Strategy not implemented for this framework");
+ }
+
/**
* A user provided filter for determining which files will be considered by the rewrite strategy.
* This will be used in addition to whatever rules the rewrite strategy generates. For example
diff --git a/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java b/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
index 8d823aa804d7..ec55d599d7bf 100644
--- a/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
+++ b/api/src/main/java/org/apache/iceberg/actions/RewriteTablePath.java
@@ -62,6 +62,11 @@ public interface RewriteTablePath extends ActionManifests added before this version are not rewritten, so snapshots that still reference
+ * them keep the {@code manifest_length} recorded in the source table. That length does not match
+ * the manifest at the target when the source and target prefixes differ in length, and readers
+ * that validate the field will reject those manifest lists.
+ *
* @param startVersion name of a metadata.json file. For example,
* "00001-8893aa9e-f92e-4443-80e7-cfa42238a654.metadata.json".
* @return this for method chaining
diff --git a/api/src/main/java/org/apache/iceberg/actions/SnapshotTable.java b/api/src/main/java/org/apache/iceberg/actions/SnapshotTable.java
index a28e94bcdbe5..1c9975242bdd 100644
--- a/api/src/main/java/org/apache/iceberg/actions/SnapshotTable.java
+++ b/api/src/main/java/org/apache/iceberg/actions/SnapshotTable.java
@@ -69,6 +69,18 @@ default SnapshotTable executeWith(ExecutorService service) {
throw new UnsupportedOperationException("Setting executor service is not supported");
}
+ /**
+ * Enables ignoring {@link java.io.FileNotFoundException} when listing source data files. When
+ * enabled, source data files that have disappeared (for example, because a partition directory
+ * was removed by concurrent cleanup) are skipped with a warning instead of failing the snapshot.
+ * The default is to fail.
+ *
+ * @return this for method chaining
+ */
+ default SnapshotTable ignoreMissingFiles() {
+ throw new UnsupportedOperationException("Ignoring missing files is not supported");
+ }
+
/** The action result that contains a summary of the execution. */
interface Result {
/** Returns the number of imported data files. */
diff --git a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
index 3438f445ec2a..fddae97125f5 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/Catalog.java
@@ -325,6 +325,20 @@ default boolean dropTable(TableIdentifier identifier) {
*/
Table loadTable(TableIdentifier identifier);
+ /**
+ * Load a table, passing the context it is being loaded with.
+ *
+ * The default implementation ignores the context.
+ *
+ * @param identifier a table identifier
+ * @param context context for this load
+ * @return instance of {@link Table} implementation referred by {@code identifier}
+ * @throws NoSuchTableException if the table does not exist
+ */
+ default Table loadTable(TableIdentifier identifier, LoadContext context) {
+ return loadTable(identifier);
+ }
+
/**
* Invalidate cached table metadata from current catalog.
*
diff --git a/api/src/main/java/org/apache/iceberg/catalog/CatalogObjectIdentifier.java b/api/src/main/java/org/apache/iceberg/catalog/CatalogObjectIdentifier.java
new file mode 100644
index 000000000000..674e519b22ac
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/catalog/CatalogObjectIdentifier.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.catalog;
+
+import java.util.Arrays;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+import org.apache.iceberg.relocated.com.google.common.base.Joiner;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+
+/**
+ * A reference to a catalog object as an ordered list of hierarchical levels (for example, a table,
+ * view, or namespace). The kind of object is determined by context — the endpoint or a companion
+ * type discriminator — not by the identifier structure alone.
+ *
+ *
Mirrors {@link Namespace} structurally; the distinct name signals "any object within a
+ * catalog" and avoids confusion with a future top-level catalog name.
+ */
+public class CatalogObjectIdentifier {
+ private static final Joiner DOT = Joiner.on('.');
+ private static final Predicate CONTAINS_NULL_CHARACTER =
+ Pattern.compile("\u0000", Pattern.UNICODE_CHARACTER_CLASS).asPredicate();
+
+ public static CatalogObjectIdentifier of(String... levels) {
+ return new CatalogObjectIdentifier(levels);
+ }
+
+ private final String[] levels;
+
+ private CatalogObjectIdentifier(String[] levels) {
+ Preconditions.checkArgument(
+ null != levels, "Cannot create catalog object identifier from null array");
+
+ for (String level : levels) {
+ Preconditions.checkNotNull(
+ level, "Cannot create a catalog object identifier with a null level");
+ Preconditions.checkArgument(
+ !CONTAINS_NULL_CHARACTER.test(level),
+ "Cannot create a catalog object identifier with the null-byte character");
+ }
+
+ this.levels = levels;
+ }
+
+ public String[] levels() {
+ return levels;
+ }
+
+ public String level(int pos) {
+ return levels[pos];
+ }
+
+ public int length() {
+ return levels.length;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other == null || getClass() != other.getClass()) {
+ return false;
+ }
+
+ CatalogObjectIdentifier that = (CatalogObjectIdentifier) other;
+ return Arrays.equals(levels, that.levels);
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(levels);
+ }
+
+ @Override
+ public String toString() {
+ return DOT.join(levels);
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/catalog/LoadContext.java b/api/src/main/java/org/apache/iceberg/catalog/LoadContext.java
new file mode 100644
index 000000000000..af31a37282fe
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/catalog/LoadContext.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.catalog;
+
+import java.util.List;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+
+/**
+ * Caller-supplied details about a load request, such as the chain of views that reference the table
+ * or view being loaded.
+ *
+ * A catalog may use these fields to make authorization, credential-scoping, and auditing
+ * decisions. All fields are optional; {@link #empty()} carries none.
+ */
+public final class LoadContext {
+ private static final LoadContext EMPTY = builder().build();
+
+ private final List referencedBy;
+
+ private LoadContext(Builder builder) {
+ this.referencedBy =
+ builder.referencedBy != null
+ ? ImmutableList.copyOf(builder.referencedBy)
+ : ImmutableList.of();
+ }
+
+ /**
+ * Returns the view identifiers that form the reference chain, outermost first.
+ *
+ * For example, if view A references view B which references table C, loading C would have
+ * {@code referencedBy = [A, B]}.
+ *
+ * @return an unmodifiable list of referencing view identifiers, or an empty list if none
+ */
+ public List referencedBy() {
+ return referencedBy;
+ }
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static LoadContext empty() {
+ return EMPTY;
+ }
+
+ public static final class Builder {
+ private List referencedBy;
+
+ private Builder() {}
+
+ public Builder referencedBy(List views) {
+ this.referencedBy = views;
+ return this;
+ }
+
+ public LoadContext build() {
+ return new LoadContext(this);
+ }
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
index b54570485357..a3e269613591 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/SessionCatalog.java
@@ -222,6 +222,21 @@ default boolean tableExists(SessionContext context, TableIdentifier ident) {
*/
Table loadTable(SessionContext context, TableIdentifier ident);
+ /**
+ * Load a table, passing the context it is being loaded with.
+ *
+ * The default implementation ignores the context.
+ *
+ * @param context session context
+ * @param ident a table identifier
+ * @param loadContext context for this load
+ * @return instance of {@link Table} implementation referred by {@code ident}
+ * @throws NoSuchTableException if the table does not exist
+ */
+ default Table loadTable(SessionContext context, TableIdentifier ident, LoadContext loadContext) {
+ return loadTable(context, ident);
+ }
+
/**
* Drop a table, without requesting that files are immediately deleted.
*
@@ -363,6 +378,7 @@ default List listNamespaces(SessionContext context) {
* @param namespace a {@link Namespace namespace}
* @param updates properties to set for the namespace
* @param removals properties to remove from the namespace
+ * @return true if the namespace metadata was successfully updated, false otherwise
* @throws NoSuchNamespaceException If the namespace does not exist (optional)
* @throws UnsupportedOperationException If namespace properties are not supported
*/
diff --git a/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java b/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java
index 7c3af5fe5745..fda96a324e9e 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java
@@ -129,6 +129,7 @@ default List listNamespaces() {
*
* @param namespace a namespace. {@link Namespace}
* @param properties a collection of metadata to apply to the namespace
+ * @return true if the properties were successfully set, false otherwise
* @throws NoSuchNamespaceException If the namespace does not exist (optional)
* @throws UnsupportedOperationException If namespace properties are not supported
*/
@@ -142,6 +143,7 @@ boolean setProperties(Namespace namespace, Map properties)
*
* @param namespace a namespace. {@link Namespace}
* @param properties a collection of metadata to apply to the namespace
+ * @return true if the properties were successfully removed, false otherwise
* @throws NoSuchNamespaceException If the namespace does not exist (optional)
* @throws UnsupportedOperationException If namespace properties are not supported
*/
diff --git a/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
index 52a715f807fa..30906d23fa55 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/ViewCatalog.java
@@ -54,6 +54,20 @@ public interface ViewCatalog {
*/
View loadView(TableIdentifier identifier);
+ /**
+ * Load a view, passing the context it is being loaded with.
+ *
+ * The default implementation ignores the context.
+ *
+ * @param identifier a view identifier
+ * @param context context for this load
+ * @return instance of {@link View} implementation referred by the identifier
+ * @throws NoSuchViewException if the view does not exist
+ */
+ default View loadView(TableIdentifier identifier, LoadContext context) {
+ return loadView(identifier);
+ }
+
/**
* Check whether view exists.
*
diff --git a/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
index 0e195665f3c2..118c9fdcc63b 100644
--- a/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
+++ b/api/src/main/java/org/apache/iceberg/catalog/ViewSessionCatalog.java
@@ -54,6 +54,22 @@ public interface ViewSessionCatalog {
*/
View loadView(SessionCatalog.SessionContext context, TableIdentifier identifier);
+ /**
+ * Load a view, passing the context it is being loaded with.
+ *
+ *
The default implementation ignores the context.
+ *
+ * @param context session context
+ * @param identifier a view identifier
+ * @param loadContext context for this load
+ * @return instance of {@link View} implementation referred by the identifier
+ * @throws NoSuchViewException if the view does not exist
+ */
+ default View loadView(
+ SessionCatalog.SessionContext context, TableIdentifier identifier, LoadContext loadContext) {
+ return loadView(context, identifier);
+ }
+
/**
* Check whether view exists.
*
diff --git a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
index 575171f85268..6a14db3dd439 100644
--- a/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
+++ b/api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java
@@ -23,6 +23,7 @@
import java.io.Serializable;
import java.io.UncheckedIOException;
import java.nio.ByteBuffer;
+import java.util.List;
import java.util.Map;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
@@ -37,7 +38,7 @@
import org.apache.iceberg.io.OutputFile;
import org.apache.iceberg.io.SupportsPrefixOperations;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
public class EncryptingFileIO implements FileIO, Serializable {
public static EncryptingFileIO combine(FileIO io, EncryptionManager em) {
@@ -68,11 +69,21 @@ public static EncryptingFileIO combine(FileIO io, EncryptionManager em) {
}
public Map bulkDecrypt(Iterable extends ContentFile>> files) {
- Iterable decrypted = em.decrypt(Iterables.transform(files, this::wrap));
-
ImmutableMap.Builder builder = ImmutableMap.builder();
- for (InputFile in : decrypted) {
- builder.put(in.location(), in);
+
+ List encryptedFiles = Lists.newArrayList();
+ for (ContentFile> file : files) {
+ if (file.keyMetadata() != null) {
+ encryptedFiles.add(wrap(file));
+ } else {
+ builder.put(file.location(), io.newInputFile(file.location(), file.fileSizeInBytes()));
+ }
+ }
+
+ if (!encryptedFiles.isEmpty()) {
+ for (InputFile file : em.decrypt(encryptedFiles)) {
+ builder.put(file.location(), file);
+ }
}
return builder.buildKeepingLast();
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
index 025b723a21cf..8f77bd7b0562 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
@@ -58,9 +58,9 @@ public Set literalSet() {
public boolean test(T value) {
switch (op()) {
case IN:
- return literalSet.contains(value);
+ return value != null && literalSet.contains(value);
case NOT_IN:
- return !literalSet.contains(value);
+ return value == null || !literalSet.contains(value);
default:
throw new IllegalStateException("Invalid operation for BoundSetPredicate: " + op());
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java b/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java
index 96e148a2d438..524d636566bb 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Evaluator.java
@@ -138,7 +138,8 @@ public Boolean notEq(Bound valueExpr, Literal lit) {
@Override
public Boolean in(Bound valueExpr, Set literalSet) {
- return literalSet.contains(valueExpr.eval(struct));
+ T value = valueExpr.eval(struct);
+ return value != null && literalSet.contains(value);
}
@Override
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expression.java b/api/src/main/java/org/apache/iceberg/expressions/Expression.java
index 4a047e08099f..9ebd1df75419 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Expression.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Expression.java
@@ -53,7 +53,7 @@ enum Operation {
public static Operation fromString(String operationType) {
Preconditions.checkArgument(null != operationType, "Invalid operation type: null");
try {
- return Expression.Operation.valueOf(operationType.toUpperCase(Locale.ENGLISH));
+ return Expression.Operation.valueOf(operationType.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException(
String.format("Invalid operation type: %s", operationType), e);
diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java
new file mode 100644
index 000000000000..4afed0624f52
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveEvalVisitor.java
@@ -0,0 +1,518 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.expressions;
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Set;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.types.Comparators;
+import org.apache.iceberg.util.NaNUtil;
+
+abstract class InclusiveEvalVisitor extends ExpressionVisitors.BoundVisitor {
+ private static final int IN_PREDICATE_LIMIT = 200;
+
+ protected static final boolean ROWS_MIGHT_MATCH = true;
+ protected static final boolean ROWS_CANNOT_MATCH = false;
+
+ /** Return true if null count is non-zero or is unknown, false if null count is 0. */
+ protected abstract boolean mayContainNull(int id);
+
+ /** Return true if null count is known and equal to value count, false otherwise. */
+ protected abstract boolean containsNullsOnly(int id);
+
+ /** Return true if NaN count is non-zero or is unknown, false if NaN count is 0. */
+ protected abstract boolean mayContainNaN(int id);
+
+ /** Return true if NaN count is known and equal to value count, false otherwise. */
+ protected abstract boolean containsNaNsOnly(int id);
+
+ /** Return the lower bound if it is known, or null otherwise. */
+ protected T lowerBound(BoundReference ref) {
+ return null;
+ }
+
+ /** Return the upper bound if it is known, or null otherwise. */
+ protected T upperBound(BoundReference ref) {
+ return null;
+ }
+
+ /** Return a variant field's lower bound if it is known, or null otherwise. */
+ protected T extractLowerBound(BoundExtract bound) {
+ return null;
+ }
+
+ /** Return a variant field's upper bound if it is known, or null otherwise. */
+ protected T extractUpperBound(BoundExtract bound) {
+ return null;
+ }
+
+ @Override
+ public Boolean alwaysTrue() {
+ return ROWS_MIGHT_MATCH; // all rows match
+ }
+
+ @Override
+ public Boolean alwaysFalse() {
+ return ROWS_CANNOT_MATCH; // all rows fail
+ }
+
+ @Override
+ public Boolean not(Boolean result) {
+ return !result;
+ }
+
+ @Override
+ public Boolean and(Boolean leftResult, Boolean rightResult) {
+ return leftResult && rightResult;
+ }
+
+ @Override
+ public Boolean or(Boolean leftResult, Boolean rightResult) {
+ return leftResult || rightResult;
+ }
+
+ @Override
+ public Boolean isNull(Bound term) {
+ // no need to check whether the field is required because binding evaluates that case
+ // if the column has no null values, the expression cannot match
+ if (isNonNullPreserving(term)) {
+ // number of non-nulls is the same as for the ref
+ int id = term.ref().fieldId();
+ if (!mayContainNull(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean notNull(Bound term) {
+ // no need to check whether the field is required because binding evaluates that case
+ // if the column has no non-null values, the expression cannot match
+
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean isNaN(Bound term) {
+ // when there's no nanCounts information, but we already know the column only contains null,
+ // it's guaranteed that there's no NaN value
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ if (!(term instanceof BoundReference)) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ if (!mayContainNaN(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean notNaN(Bound term) {
+ if (!(term instanceof BoundReference)) {
+ // identity transforms are already removed by this time
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int id = term.ref().fieldId();
+
+ if (containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean lt(Bound term, Literal lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // this also works for transforms that are order preserving:
+ // if a transform f is order preserving, a < b means that f(a) <= f(b).
+ // because lower <= a for all values of a in the file, f(lower) <= f(a).
+ // when f(lower) >= X then f(a) >= f(lower) >= X, so there is no a such that f(a) < X
+ // f(lower) >= X means rows cannot match
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp >= 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean ltEq(Bound term, Literal lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // this also works for transforms that are order preserving:
+ // if a transform f is order preserving, a < b means that f(a) <= f(b).
+ // because lower <= a for all values of a in the file, f(lower) <= f(a).
+ // when f(lower) > X then f(a) >= f(lower) > X, so there is no a such that f(a) <= X
+ // f(lower) > X means rows cannot match
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean gt(Bound term, Literal lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp <= 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean gtEq(Bound term, Literal lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean eq(Bound term, Literal lit) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (lower != null && !NaNUtil.isNaN(lower)) {
+ int cmp = lit.comparator().compare(lower, lit.value());
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ T upper = evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = lit.comparator().compare(upper, lit.value());
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean notEq(Bound term, Literal lit) {
+ // because the bounds are not necessarily a min or max value, this cannot be answered using
+ // them. notEq(col, X) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value equals the literal.
+ T value = uniqueValue(term);
+
+ if (value != null && lit.comparator().compare(value, lit.value()) == 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean in(Bound term, Set literalSet) {
+ // all terms are null preserving. see #isNullPreserving(Bound)
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id) || containsNaNsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ Collection literals = literalSet;
+
+ if (literals.size() > IN_PREDICATE_LIMIT) {
+ // skip evaluating the predicate if the number of values is too big
+ return ROWS_MIGHT_MATCH;
+ }
+
+ T lower = evalLowerBound(term);
+ if (null == lower || NaNUtil.isNaN(lower)) {
+ // NaN indicates unreliable bounds. See the InclusiveMetricsEvaluator docs for more.
+ return ROWS_MIGHT_MATCH;
+ }
+
+ Comparator comparator = ((BoundTerm) term).comparator();
+ T upper = evalUpperBound(term);
+
+ // rows might match if any literal falls within [lower, upper]; a null upper means no upper
+ // bound
+ for (T literal : literals) {
+ if (comparator.compare(lower, literal) <= 0
+ && (null == upper || comparator.compare(upper, literal) >= 0)) {
+ return ROWS_MIGHT_MATCH;
+ }
+ }
+
+ return ROWS_CANNOT_MATCH;
+ }
+
+ @Override
+ public Boolean notIn(Bound term, Set literalSet) {
+ // because the bounds are not necessarily a min or max value, this cannot be answered using
+ // them. notIn(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value in col.
+ // However, when min == max and the file has no nulls or NaN values, we can safely prune
+ // if that value is in the exclusion set.
+ T value = uniqueValue(term);
+
+ if (value != null && literalSet.contains(value)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean startsWith(Bound term, Literal lit) {
+ if (term instanceof BoundTransform && !((BoundTransform, ?>) term).transform().isIdentity()) {
+ // truncate must be rewritten in binding. the result is either always or never compatible
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int id = term.ref().fieldId();
+ if (containsNullsOnly(id)) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ String prefix = (String) lit.value();
+
+ Comparator comparator = Comparators.charSequences();
+
+ CharSequence lower = (CharSequence) evalLowerBound(term);
+ if (null == lower) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // truncate lower bound so that its length in bytes is not greater than the length of prefix
+ int length = Math.min(prefix.length(), lower.length());
+ int cmp = comparator.compare(lower.subSequence(0, length), prefix);
+ if (cmp > 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ CharSequence upper = (CharSequence) evalUpperBound(term);
+ if (null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // truncate upper bound so that its length in bytes is not greater than the length of prefix
+ length = Math.min(prefix.length(), upper.length());
+ cmp = comparator.compare(upper.subSequence(0, length), prefix);
+ if (cmp < 0) {
+ return ROWS_CANNOT_MATCH;
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ @Override
+ public Boolean notStartsWith(Bound term, Literal lit) {
+ // the only transforms that produce strings are truncate and identity, which work with this
+ int id = term.ref().fieldId();
+ if (mayContainNull(id)) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ String prefix = (String) lit.value();
+
+ Comparator comparator = Comparators.charSequences();
+
+ // notStartsWith will match unless all values must start with the prefix. This happens when
+ // the lower and upper bounds both start with the prefix.
+ CharSequence lower = (CharSequence) evalLowerBound(term);
+ CharSequence upper = (CharSequence) evalUpperBound(term);
+ if (null == lower || null == upper) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ // if lower is shorter than the prefix then lower doesn't start with the prefix
+ if (lower.length() < prefix.length()) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ int cmp = comparator.compare(lower.subSequence(0, prefix.length()), prefix);
+ if (cmp == 0) {
+ // if upper is shorter than the prefix then upper can't start with the prefix
+ if (upper.length() < prefix.length()) {
+ return ROWS_MIGHT_MATCH;
+ }
+
+ cmp = comparator.compare(upper.subSequence(0, prefix.length()), prefix);
+ if (cmp == 0) {
+ // both bounds match the prefix, so all rows must match the prefix and therefore do not
+ // satisfy the predicate
+ return ROWS_CANNOT_MATCH;
+ }
+ }
+
+ return ROWS_MIGHT_MATCH;
+ }
+
+ /**
+ * Returns the column's single value if all rows contain the same value. Defined as a column with
+ * no nulls, no NaNs, and lower bound equals upper bound. Returns null otherwise.
+ */
+ private T uniqueValue(Bound term) {
+ int id = term.ref().fieldId();
+ if (mayContainNull(id)) {
+ return null;
+ }
+
+ T lower = evalLowerBound(term);
+ T upper = evalUpperBound(term);
+
+ if (lower == null || upper == null || NaNUtil.isNaN(lower) || NaNUtil.isNaN(upper)) {
+ return null;
+ }
+
+ if ((lower instanceof Float || lower instanceof Double) && mayContainNaN(id)) {
+ return null;
+ }
+
+ if (!lower.equals(upper)) {
+ return null;
+ }
+
+ return lower;
+ }
+
+ private T evalLowerBound(Bound term) {
+ if (term instanceof BoundReference) {
+ return lowerBound((BoundReference) term);
+ } else if (term instanceof BoundTransform) {
+ return transformLowerBound((BoundTransform, T>) term);
+ } else if (term instanceof BoundExtract) {
+ return extractLowerBound((BoundExtract) term);
+ } else {
+ return null;
+ }
+ }
+
+ private T evalUpperBound(Bound term) {
+ if (term instanceof BoundReference) {
+ return upperBound((BoundReference) term);
+ } else if (term instanceof BoundTransform) {
+ return transformUpperBound((BoundTransform, T>) term);
+ } else if (term instanceof BoundExtract) {
+ return extractUpperBound((BoundExtract) term);
+ } else {
+ return null;
+ }
+ }
+
+ private T transformLowerBound(BoundTransform boundTransform) {
+ Transform transform = boundTransform.transform();
+ if (transform.preservesOrder()) {
+ S lower = lowerBound(boundTransform.ref());
+ return boundTransform.transform().bind(boundTransform.ref().type()).apply(lower);
+ }
+
+ return null;
+ }
+
+ private T transformUpperBound(BoundTransform boundTransform) {
+ Transform transform = boundTransform.transform();
+ if (transform.preservesOrder()) {
+ S upper = upperBound(boundTransform.ref());
+ return boundTransform.transform().bind(boundTransform.ref().type()).apply(upper);
+ }
+
+ return null;
+ }
+
+ /** Returns true if the expression term produces a non-null value for non-null input. */
+ private boolean isNonNullPreserving(Bound> term) {
+ if (term instanceof BoundReference) {
+ return true;
+ } else if (term instanceof BoundTransform, ?>) {
+ // if the transform preserves order, then non-null values are mapped to non-null
+ return ((BoundTransform, ?>) term).transform().preservesOrder();
+ }
+
+ // a non-null variant does not necessarily contain a specific field
+ // and unknown bound terms are not non-null preserving
+ return false;
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
index 81cbbe785519..01295e21e305 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/InclusiveMetricsEvaluator.java
@@ -21,19 +21,12 @@
import static org.apache.iceberg.expressions.Expressions.rewriteNot;
import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Comparator;
import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Schema;
-import org.apache.iceberg.transforms.Transform;
-import org.apache.iceberg.types.Comparators;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types.StructType;
-import org.apache.iceberg.util.NaNUtil;
import org.apache.iceberg.variants.Variant;
import org.apache.iceberg.variants.VariantObject;
@@ -53,8 +46,6 @@
* checks for NaN is necessary in order to not skip files that may contain matching data.
*/
public class InclusiveMetricsEvaluator {
- private static final int IN_PREDICATE_LIMIT = 200;
-
private final Expression expr;
public InclusiveMetricsEvaluator(Schema schema, Expression unbound) {
@@ -77,10 +68,7 @@ public boolean eval(ContentFile> file) {
return new MetricsEvalVisitor().eval(file);
}
- private static final boolean ROWS_MIGHT_MATCH = true;
- private static final boolean ROWS_CANNOT_MATCH = false;
-
- private class MetricsEvalVisitor extends ExpressionVisitors.BoundVisitor {
+ private class MetricsEvalVisitor extends InclusiveEvalVisitor {
private Map valueCounts = null;
private Map nullCounts = null;
private Map nanCounts = null;
@@ -109,389 +97,12 @@ private boolean eval(ContentFile> file) {
}
@Override
- public Boolean alwaysTrue() {
- return ROWS_MIGHT_MATCH; // all rows match
- }
-
- @Override
- public Boolean alwaysFalse() {
- return ROWS_CANNOT_MATCH; // all rows fail
- }
-
- @Override
- public Boolean not(Boolean result) {
- return !result;
- }
-
- @Override
- public Boolean and(Boolean leftResult, Boolean rightResult) {
- return leftResult && rightResult;
- }
-
- @Override
- public Boolean or(Boolean leftResult, Boolean rightResult) {
- return leftResult || rightResult;
- }
-
- @Override
- public Boolean isNull(Bound term) {
- // no need to check whether the field is required because binding evaluates that case
- // if the column has no null values, the expression cannot match
- if (isNonNullPreserving(term)) {
- // number of non-nulls is the same as for the ref
- int id = term.ref().fieldId();
- if (!mayContainNull(id)) {
- return ROWS_CANNOT_MATCH;
- }
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public Boolean notNull(Bound term) {
- // no need to check whether the field is required because binding evaluates that case
- // if the column has no non-null values, the expression cannot match
-
- // all terms are null preserving. see #isNullPreserving(Bound)
- int id = term.ref().fieldId();
- if (containsNullsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public Boolean isNaN(Bound term) {
- // when there's no nanCounts information, but we already know the column only contains null,
- // it's guaranteed that there's no NaN value
- int id = term.ref().fieldId();
- if (containsNullsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- if (!(term instanceof BoundReference)) {
- return ROWS_MIGHT_MATCH;
- }
-
- if (nanCounts != null && nanCounts.containsKey(id) && nanCounts.get(id) == 0) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public Boolean notNaN(Bound term) {
- if (!(term instanceof BoundReference)) {
- // identity transforms are already removed by this time
- return ROWS_MIGHT_MATCH;
- }
-
- int id = term.ref().fieldId();
-
- if (containsNaNsOnly(id)) {
- return ROWS_CANNOT_MATCH;
- }
-
- return ROWS_MIGHT_MATCH;
- }
-
- @Override
- public Boolean lt(Bound term, Literal