Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions api/src/main/java/org/apache/iceberg/util/TruncateUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,15 +45,15 @@ public class TruncateUtil {
private TruncateUtil() {}

public static byte truncateByte(int width, byte value) {
return (byte) (value - (((value % width) + width) % width));
return (byte) (value - Math.floorMod(value, width));

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

1. The remainder, `v % W`, must be positive. For languages where `%` can produce negative values, the correct truncate function is: `v - (((v % W) + W) % W)`

This fixes the implementation, but the spec above still doesn’t say the arithmetic must use unbounded integers. Could we clarify the spec as r = floorMod(v, W) and v - r computed with unbounded arithmetic? Otherwise other implementations can still produce different partition values.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another issue is that truncateInt(10, Integer.MIN_VALUE) still returns Integer.MAX_VALUE - 1 after this PR: the final mathematical result is outside int, so wrapping breaks ordering and can cause incorrect partition pruning as reported in #9767 and #13105.

Do we need a spec change for clarification?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

+1 to updating the spec accordingly.

}

public static short truncateShort(int width, short value) {
return (short) (value - (((value % width) + width) % width));
return (short) (value - Math.floorMod(value, width));
}

public static int truncateInt(int width, int value) {
return value - (((value % width) + width) % width);
return value - Math.floorMod(value, width);
}

public static long truncateLong(int width, long value) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.iceberg;

import static org.assertj.core.api.Assertions.assertThat;

import java.util.Map;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.inmemory.InMemoryCatalog;
import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.types.Types;
import org.junit.jupiter.api.Test;

class TestTruncatePartitionScanning {
@Test
void largeWidthRetainsMatchingPartition() throws Exception {
Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get()));
PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("id", Integer.MAX_VALUE).build();
PartitionKey key = new PartitionKey(spec, schema);
key.partition(TestHelpers.Row.of(Integer.MAX_VALUE - 1));

try (InMemoryCatalog catalog = new InMemoryCatalog()) {
catalog.initialize("test", Map.of());
catalog.createNamespace(Namespace.of("db"));
Table table = catalog.createTable(TableIdentifier.of("db", "table"), schema, spec);
DataFile file =
DataFiles.builder(spec)
.withPath("/data/file.parquet")
.withPartition(key)
.withFileSizeInBytes(100)
.withRecordCount(1)
.build();
table.newAppend().appendFile(file).commit();

try (CloseableIterable<FileScanTask> tasks =
table.newScan().filter(Expressions.greaterThanOrEqual("id", 0)).planFiles()) {
assertThat(tasks)
.extracting(task -> task.file().location())
.containsExactly(file.location());
}
}
}
}
17 changes: 17 additions & 0 deletions core/src/test/java/org/apache/iceberg/util/TestTruncateUtil.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,29 @@
*/
package org.apache.iceberg.util;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatException;
import static org.assertj.core.api.Assertions.assertThatNoException;

import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

class TestTruncateUtil {
@ParameterizedTest
@ValueSource(ints = {1_500_000_000, 2_000_000_000, Integer.MAX_VALUE})
void largeIntegerWidth(int width) {
assertThat(TruncateUtil.truncateInt(width, width - 1)).isZero();
assertThat(TruncateUtil.truncateInt(width, width)).isEqualTo(width);
assertThat(TruncateUtil.truncateInt(width, -1)).isEqualTo(-width);
}

@Test
void largeWidthForNarrowIntegers() {
assertThat(TruncateUtil.truncateByte(Integer.MAX_VALUE, Byte.MAX_VALUE)).isZero();
assertThat(TruncateUtil.truncateShort(Integer.MAX_VALUE, Short.MAX_VALUE)).isZero();
}

@Test
public void testInvalidInputWidthBehavior() {
assertThatNoException()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,20 @@ public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}

@TestTemplate
void scanWithLargeIntegerTruncateWidth() {
sql(
"CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) "
+ "TBLPROPERTIES('%s' = '%s')",
tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format);
int value = Integer.MAX_VALUE - 1;
sql("INSERT INTO %s VALUES (%s)", tableName, value);

assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName))
.singleElement()
.satisfies(row -> assertThat(row[0]).isEqualTo(value));
}

@TestTemplate
public void testEstimatedRowCount() throws NoSuchTableException {
sql(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,20 @@ public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}

@TestTemplate
void scanWithLargeIntegerTruncateWidth() {
sql(
"CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) "
+ "TBLPROPERTIES('%s' = '%s')",
tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format);
int value = Integer.MAX_VALUE - 1;
sql("INSERT INTO %s VALUES (%s)", tableName, value);

assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName))
.singleElement()
.satisfies(row -> assertThat(row[0]).isEqualTo(value));
}

@TestTemplate
public void testEstimatedRowCount() throws NoSuchTableException {
sql(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,20 @@ public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}

@TestTemplate
void scanWithLargeIntegerTruncateWidth() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we add this one for the other Spark versions as well?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added the regression test for Spark 3.5, 4.0, and 4.2. All nine added cases passed across Parquet, Avro, etc. Updated validation notes too.

sql(
"CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) "
+ "TBLPROPERTIES('%s' = '%s')",
tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format);
int value = Integer.MAX_VALUE - 1;
sql("INSERT INTO %s VALUES (%s)", tableName, value);

assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName))
.singleElement()
.satisfies(row -> assertThat(row[0]).isEqualTo(value));
}

@TestTemplate
public void testEstimatedRowCount() throws NoSuchTableException {
sql(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,20 @@ public void removeTables() {
sql("DROP TABLE IF EXISTS %s", tableName);
}

@TestTemplate
void scanWithLargeIntegerTruncateWidth() {
sql(
"CREATE TABLE %s (id INT) USING iceberg PARTITIONED BY (truncate(%s, id)) "
+ "TBLPROPERTIES('%s' = '%s')",
tableName, Integer.MAX_VALUE, TableProperties.DEFAULT_FILE_FORMAT, format);
int value = Integer.MAX_VALUE - 1;
sql("INSERT INTO %s VALUES (%s)", tableName, value);

assertThat(sql("SELECT id FROM %s WHERE id >= 0", tableName))
.singleElement()
.satisfies(row -> assertThat(row[0]).isEqualTo(value));
}

@TestTemplate
public void testEstimatedRowCount() throws NoSuchTableException {
sql(
Expand Down
Loading