diff --git a/api/src/main/java/org/apache/iceberg/util/TruncateUtil.java b/api/src/main/java/org/apache/iceberg/util/TruncateUtil.java index 8b53d05e88bb..36e748da1681 100644 --- a/api/src/main/java/org/apache/iceberg/util/TruncateUtil.java +++ b/api/src/main/java/org/apache/iceberg/util/TruncateUtil.java @@ -45,15 +45,15 @@ public class TruncateUtil { private TruncateUtil() {} public static byte truncateByte(int width, byte value) { - return (byte) (value - (((value % width) + width) % width)); + return (byte) (value - Math.floorMod(value, width)); } public static short truncateShort(int width, short value) { - return (short) (value - (((value % width) + width) % width)); + return (short) (value - Math.floorMod(value, width)); } public static int truncateInt(int width, int value) { - return value - (((value % width) + width) % width); + return value - Math.floorMod(value, width); } public static long truncateLong(int width, long value) { diff --git a/core/src/test/java/org/apache/iceberg/TestTruncatePartitionScanning.java b/core/src/test/java/org/apache/iceberg/TestTruncatePartitionScanning.java new file mode 100644 index 000000000000..57826fc1ef90 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestTruncatePartitionScanning.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.inmemory.InMemoryCatalog; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Test; + +class TestTruncatePartitionScanning { + @Test + void largeWidthRetainsMatchingPartition() throws Exception { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("id", Integer.MAX_VALUE).build(); + PartitionKey key = new PartitionKey(spec, schema); + key.partition(TestHelpers.Row.of(Integer.MAX_VALUE - 1)); + + try (InMemoryCatalog catalog = new InMemoryCatalog()) { + catalog.initialize("test", Map.of()); + catalog.createNamespace(Namespace.of("db")); + Table table = catalog.createTable(TableIdentifier.of("db", "table"), schema, spec); + DataFile file = + DataFiles.builder(spec) + .withPath("/data/file.parquet") + .withPartition(key) + .withFileSizeInBytes(100) + .withRecordCount(1) + .build(); + table.newAppend().appendFile(file).commit(); + + try (CloseableIterable tasks = + table.newScan().filter(Expressions.greaterThanOrEqual("id", 0)).planFiles()) { + assertThat(tasks) + .extracting(task -> task.file().location()) + .containsExactly(file.location()); + } + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/util/TestTruncateUtil.java b/core/src/test/java/org/apache/iceberg/util/TestTruncateUtil.java index 025f0c96968d..5c34688435ca 100644 --- a/core/src/test/java/org/apache/iceberg/util/TestTruncateUtil.java +++ b/core/src/test/java/org/apache/iceberg/util/TestTruncateUtil.java @@ -18,12 +18,29 @@ */ package org.apache.iceberg.util; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatException; import static org.assertj.core.api.Assertions.assertThatNoException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; class TestTruncateUtil { + @ParameterizedTest + @ValueSource(ints = {1_500_000_000, 2_000_000_000, Integer.MAX_VALUE}) + void largeIntegerWidth(int width) { + assertThat(TruncateUtil.truncateInt(width, width - 1)).isZero(); + assertThat(TruncateUtil.truncateInt(width, width)).isEqualTo(width); + assertThat(TruncateUtil.truncateInt(width, -1)).isEqualTo(-width); + } + + @Test + void largeWidthForNarrowIntegers() { + assertThat(TruncateUtil.truncateByte(Integer.MAX_VALUE, Byte.MAX_VALUE)).isZero(); + assertThat(TruncateUtil.truncateShort(Integer.MAX_VALUE, Short.MAX_VALUE)).isZero(); + } + @Test public void testInvalidInputWidthBehavior() { assertThatNoException() diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java index 2eb61407b0f0..222578270e73 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java @@ -122,6 +122,20 @@ public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } + @TestTemplate + void scanWithLargeIntegerTruncateWidth() { + sql( + "CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) " + + "TBLPROPERTIES('%s' = '%s')", + tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format); + int value = Integer.MAX_VALUE - 1; + sql("INSERT INTO %s VALUES (%s)", tableName, value); + + assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName)) + .singleElement() + .satisfies(row -> assertThat(row[0]).isEqualTo(value)); + } + @TestTemplate public void testEstimatedRowCount() throws NoSuchTableException { sql( diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java index 2eb61407b0f0..222578270e73 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java @@ -122,6 +122,20 @@ public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } + @TestTemplate + void scanWithLargeIntegerTruncateWidth() { + sql( + "CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) " + + "TBLPROPERTIES('%s' = '%s')", + tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format); + int value = Integer.MAX_VALUE - 1; + sql("INSERT INTO %s VALUES (%s)", tableName, value); + + assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName)) + .singleElement() + .satisfies(row -> assertThat(row[0]).isEqualTo(value)); + } + @TestTemplate public void testEstimatedRowCount() throws NoSuchTableException { sql( diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java index b226990e6955..678952e9e76e 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java @@ -124,6 +124,20 @@ public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } + @TestTemplate + void scanWithLargeIntegerTruncateWidth() { + sql( + "CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) " + + "TBLPROPERTIES('%s' = '%s')", + tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format); + int value = Integer.MAX_VALUE - 1; + sql("INSERT INTO %s VALUES (%s)", tableName, value); + + assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName)) + .singleElement() + .satisfies(row -> assertThat(row[0]).isEqualTo(value)); + } + @TestTemplate public void testEstimatedRowCount() throws NoSuchTableException { sql( diff --git a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java index b226990e6955..678952e9e76e 100644 --- a/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +++ b/spark/v4.2/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java @@ -124,6 +124,20 @@ public void removeTables() { sql("DROP TABLE IF EXISTS %s", tableName); } + @TestTemplate + void scanWithLargeIntegerTruncateWidth() { + sql( + "CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) " + + "TBLPROPERTIES('%s' = '%s')", + tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format); + int value = Integer.MAX_VALUE - 1; + sql("INSERT INTO %s VALUES (%s)", tableName, value); + + assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName)) + .singleElement() + .satisfies(row -> assertThat(row[0]).isEqualTo(value)); + } + @TestTemplate public void testEstimatedRowCount() throws NoSuchTableException { sql(