From 9bb05915d76e63d0e935c723f90f5e52cecea09f Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 9 Nov 2026 17:23:08 -0300 Subject: [PATCH 1/5] fix: leave wide integer truncate to Spark --- docs/source/user-guide/latest/iceberg.md | 10 +++++ .../src/execution/operators/iceberg_write.rs | 21 +++++++--- .../comet/iceberg/IcebergReflection.scala | 42 +++++++++++++++++++ .../apache/comet/serde/icebergFunctions.scala | 23 ++++++++++ .../operator/CometIcebergNativeWrite.scala | 22 ++++++++++ .../CometIcebergSystemFunctionSuite.scala | 30 +++++++++++++ .../CometIcebergWriteDetectionSuite.scala | 22 ++++++++++ 7 files changed, 164 insertions(+), 6 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 2822847e71a..941a86efbb3 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -290,6 +290,16 @@ value back unchanged and Spark turns it into null only when the row is materiali would have to null it during evaluation, which changes what an enclosing predicate or hash sees. Every other `truncate` input type, and `bucket` on decimals, runs natively. +An integer `truncate` with a width above 2^30 (`1073741824`) also falls back to Spark. At those +widths Iceberg Java's `(v % W) + W` overflows a 32-bit `int`, while the [format +spec](https://github.com/apache/iceberg/blob/main/format/spec.md) defines the result as +`v - (v % W)` with a positive remainder. Every Iceberg release through 1.12.0 overflows, and a +release carrying apache/iceberg#18112 returns the spec value, so the correct answer depends on +the Iceberg jar the user runs, not on the Spark version. Comet cannot pick a shim from the Spark +profile, so it leaves those widths to Spark. The same width limit applies to native writes whose +partition spec has an integer `truncate[W]` field: the native writer computes the spec value, +which would differ from the value the user's Iceberg jar writes into the same table. + This matters most for writes. A partitioned table with the default `write.distribution-mode` (`hash`) is planned with a shuffle and a local sort keyed on the partition transforms, and with these functions native the whole sub-plan feeding the [native Iceberg writer](#writes) diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index c949b95cf66..905e0d6f9e4 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -6140,12 +6140,21 @@ mod iceberg_rust_transform_parity { } } - /// `i32::MIN`, `i64::MIN`, and widths above 2^30 are left out: Java's `TruncateUtil` wraps - /// there and iceberg-rust does not (`truncate_i32` uses `rem_euclid`, `truncate_i64` and the - /// decimal kernel subtract without wrapping and overflow in a debug build). That is an - /// iceberg-rust bug affecting the writer's own partition values, independent of these - /// kernels -- apache/iceberg-rust#3141. The wrapping cases are pinned against the JVM in the - /// kernel's own unit tests; add them here once that issue is fixed. + /// `i32::MIN`, `i64::MIN`, and widths above 2^30 are left out because the two sides + /// intentionally disagree there. + /// + /// Widths above 2^30 are a bug in Java, not in iceberg-rust: the format spec defines integer + /// truncation as `v - (v % W)` with a positive remainder, and Java's `(v % W) + W` overflows + /// a 32-bit `int`. apache/iceberg#18111 tracks the Java bug and apache/iceberg#18112 the fix. + /// Comet's kernel reproduces Java so the expression path matches every Iceberg release + /// through 1.12.0; `CometIcebergTruncate` leaves these widths to Spark, and native writes + /// fall back when the partition spec has such a field + /// (https://github.com/apache/datafusion-comet/issues/7442). + /// + /// `i32::MIN` and `i64::MIN` are the opposite case: the result does not fit the type, the + /// spec is silent, Java wraps, and iceberg-rust panics in a debug build. That is + /// apache/iceberg-rust#3141. The wrapping cases are pinned against the JVM in the kernel's + /// own unit tests; add them here once that issue is fixed. #[test] fn truncate_agrees_with_iceberg_rust() { let inputs: Vec<(&str, ArrayRef)> = vec![ diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index ff318e0dfb6..d5f5ec8b3fa 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -918,6 +918,48 @@ object IcebergReflection extends Logging { } } + /** Largest integer `truncate` width whose `(v % W) + W` cannot overflow a Java `int`. */ + private val MaxSafeIntegerTruncateWidth: Long = 1L << 30 + + private val TruncateTransform = """^truncate\[(\d+)\]$""".r + + /** + * The names of the `truncate[W]` partition fields of `spec` whose source type is an integer + * (or a date) and whose width is above 2^30. + * + * At those widths Iceberg Java's `TruncateUtil` overflows in `(v % W) + W`, while the format + * spec and iceberg-rust compute `v - (v % W)`. A native write would therefore produce a + * different partition value than the Iceberg jar the user runs. Throws on reflection failure, + * so the caller can fail closed. + */ + def truncateFieldsWithLargeIntegerWidth(spec: Any): Seq[String] = { + import scala.jdk.CollectionConverters._ + val schema = getMethod(spec.getClass, "schema").invoke(spec) + val findField = getMethod(schema.getClass, "findField", classOf[Int]) + getMethod(spec.getClass, "fields") + .invoke(spec) + .asInstanceOf[java.util.List[_]] + .asScala + .toSeq + .filter { field => + getMethod(field.getClass, "transform").invoke(field).toString match { + case TruncateTransform(width) if width.toLong > MaxSafeIntegerTruncateWidth => + val sourceId = getMethod(field.getClass, "sourceId").invoke(field).asInstanceOf[Int] + val sourceField = findField.invoke(schema, sourceId.asInstanceOf[Object]) + if (sourceField == null) { + false + } else { + // Long, string, binary and decimal truncation does not overflow at these widths. + val sourceType = getMethod(sourceField.getClass, "type").invoke(sourceField) + val typeId = getMethod(sourceType.getClass, "typeId").invoke(sourceType).toString + typeId == "INTEGER" || typeId == "DATE" + } + case _ => false + } + } + .map(field => getMethod(field.getClass, "name").invoke(field).asInstanceOf[String]) + } + /** * Gets the partition spec from an Iceberg table. */ diff --git a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala index 51d4203efe7..7157ccb0633 100644 --- a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala @@ -222,6 +222,22 @@ object CometIcebergTruncate "Decimal128(precision, scale) array cannot carry that intermediate, so a native kernel " + "would null it during evaluation and change what an enclosing predicate or hash sees." + /** + * Largest integer `truncate` width whose `(v % W) + W` cannot overflow a Java `int`. + * + * Above it the result depends on the Iceberg jar the user runs: every release through 1.12.0 + * wraps, while a release carrying apache/iceberg#18112 returns the spec's `v - (v % W)`. + * Spark decides the expected answer, so those widths stay with Spark. + * https://github.com/apache/datafusion-comet/issues/7442 + */ + private val MaxSafeIntegerWidth: Int = 1 << 30 + + /** Types whose truncation Iceberg evaluates with Java `int` arithmetic. */ + private def usesIntArithmetic(dataType: DataType): Boolean = dataType match { + case ByteType | ShortType | IntegerType | DateType => true + case _ => false + } + // Ordered after the parameter check so that `truncate(0, decimal_col)` still reports the width // problem, which is the one that changes whether Iceberg's own ArithmeticException is raised. override def getSupportLevel(expr: Expression): SupportLevel = @@ -230,6 +246,13 @@ object CometIcebergTruncate if value.dataType.isInstanceOf[DecimalType] && CometIcebergSystemFunctions.positiveIntLiteral(parameter).isDefined => Unsupported(Some(DecimalNote)) + case Some((Seq(parameter, value), _)) + if usesIntArithmetic(value.dataType) && + CometIcebergSystemFunctions.positiveIntLiteral(parameter).exists(_ > MaxSafeIntegerWidth) => + Unsupported( + Some( + s"truncate width above $MaxSafeIntegerWidth overflows Iceberg's int arithmetic, and " + + "the result depends on the Iceberg jar the user runs")) case _ => super.getSupportLevel(expr) } diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index b385668291d..d9e82d12732 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -416,6 +416,7 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { requireNoMetadataColumns, requireSupportedColumnTypes, requireNoVoidFieldWithDroppedSource, + requireNoLargeWidthIntegerTruncate, requireNoEncryptionPrefix, requireNoBloomFilterColumnsEnabled, requireRowGroupCheckMinRecordCountAtDefault, @@ -553,6 +554,27 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } } + // An integer `truncate[W]` with `W > 2^30` overflows in Iceberg Java's `(v % W) + W`, while + // iceberg-rust computes the spec's `v - (v % W)`. Native writes would therefore write a + // different partition value than the user's Iceberg jar, which decides the Spark result. + // https://github.com/apache/datafusion-comet/issues/7442 + private val requireNoLargeWidthIntegerTruncate: TriggerRule = ctx => + IcebergReflection + .getOutputSpecIdFromSparkWrite(ctx.sparkWrite) + .flatMap(IcebergReflection.getPartitionSpecById(ctx.table, _)) match { + case None => Some("could not resolve the output partition spec for truncate width checking") + case Some(spec) => + try { + IcebergReflection.truncateFieldsWithLargeIntegerWidth(spec).headOption.map { name => + s"partition field $name uses an integer truncate width above 2^30, where " + + "iceberg-java and iceberg-rust disagree" + } + } catch { + case e: Exception => + Some(s"could not inspect the output partition spec: ${e.getMessage}") + } + } + private val requireNoEncryptionPrefix: TriggerRule = ctx => ctx.properties.keys .find(_.startsWith(EncryptionPropertyPrefix)) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index 613f726767d..ec604000e21 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -135,6 +135,23 @@ class CometIcebergSystemFunctionSuite } } + test("integer truncate widths above 2^30 fall back to Spark") { + withSourceTable { + // Iceberg Java's `(v % W) + W` overflows a 32-bit `int` above this width, while the format + // spec and iceberg-rust compute `v - (v % W)`. Every Iceberg release through 1.12.0 + // overflows and a release carrying apache/iceberg#18112 does not, so Spark's expected + // answer depends on the user's Iceberg jar. Comet leaves the expression to Spark. + // https://github.com/apache/datafusion-comet/issues/7442 + checkSparkAnswerAndFallbackReason( + s"SELECT $catalog.system.truncate(2000000000, i32) FROM $source", + "truncate width above") + // Long, string and binary truncation does not overflow at any int width. + checkSparkAnswerAndOperator( + s"SELECT $catalog.system.truncate(2000000000, i64), " + + s"$catalog.system.truncate(2000000000, str) FROM $source") + } + } + test("truncate on a decimal falls back to Spark") { withSourceTable { // Iceberg's `TruncateDecimal` can return a value wider than the column's precision, which @@ -341,6 +358,19 @@ class CometIcebergSystemFunctionSuite assert(level(CometIcebergTruncate, truncateInt, Literal(10), value) == Compatible()) assert(level(CometIcebergTruncate, truncateInt, Literal(10), date).isInstanceOf[Unsupported]) + // Widths above 2^30 overflow Iceberg Java's `int` arithmetic; widths up to it are safe. + assert(level(CometIcebergTruncate, truncateInt, Literal(1 << 30), value) == Compatible()) + assert( + level(CometIcebergTruncate, truncateInt, Literal((1 << 30) + 1), value) + .isInstanceOf[Unsupported]) + // Long truncation uses long arithmetic, so the same width stays native. + val truncateLong = + Class.forName("org.apache.iceberg.spark.functions.TruncateFunction$TruncateLong") + val longValue = AttributeReference("l", LongType)() + assert( + level(CometIcebergTruncate, truncateLong, Literal((1 << 30) + 1), longValue) == + Compatible()) + // A decimal value reports its own reason, but only once the width is valid: a zero width has // to keep reporting the width, since that is what decides whether Iceberg's own // ArithmeticException is raised. diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index f2fcc3010fe..f378411eb52 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -1499,6 +1499,28 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes } } + test("fall-back: integer truncate partition width above 2^30") { + // Above this width Iceberg Java's `(v % W) + W` overflows an `int`, while the format spec + // and iceberg-rust compute `v - (v % W)`. The user's Iceberg jar decides the value that + // belongs in the table, so the native writer cannot pick one. + // https://github.com/apache/datafusion-comet/issues/7442 + withDetectionCatalog { _ => + createTable( + "part_truncate_wide", + partitionSpec = "PARTITIONED BY (truncate(2000000000, id))") + assertUnsupportedContains("part_truncate_wide", "truncate width above 2^30") + } + } + + test("Compatible when partitioned by an integer truncate at the maximum safe width") { + withDetectionCatalog { _ => + createTable( + "part_truncate_max", + partitionSpec = "PARTITIONED BY (truncate(1073741824, id))") + assertSupportLevelIs[Compatible]("part_truncate_max") + } + } + test("Compatible when partitioned by identity on a string column") { withDetectionCatalog { _ => createTable("part_string", partitionSpec = "PARTITIONED BY (region)") From 5abc4cb34798e6649a79624beefbb429466f124c Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 9 Nov 2026 17:29:11 -0300 Subject: [PATCH 2/5] style: wrap the truncate support-level guard --- .../main/scala/org/apache/comet/serde/icebergFunctions.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala index 7157ccb0633..b12fbb3f255 100644 --- a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala @@ -248,7 +248,9 @@ object CometIcebergTruncate Unsupported(Some(DecimalNote)) case Some((Seq(parameter, value), _)) if usesIntArithmetic(value.dataType) && - CometIcebergSystemFunctions.positiveIntLiteral(parameter).exists(_ > MaxSafeIntegerWidth) => + CometIcebergSystemFunctions + .positiveIntLiteral(parameter) + .exists(_ > MaxSafeIntegerWidth) => Unsupported( Some( s"truncate width above $MaxSafeIntegerWidth overflows Iceberg's int arithmetic, and " + From 964ff857f62c90e527a2d5e3070d7e0a5211a0c7 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 9 Nov 2026 17:33:46 -0300 Subject: [PATCH 3/5] style: apply scalafmt --- .../org/apache/comet/iceberg/IcebergReflection.scala | 4 ++-- .../org/apache/comet/serde/icebergFunctions.scala | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index d5f5ec8b3fa..a5c2e5ec9cb 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -924,8 +924,8 @@ object IcebergReflection extends Logging { private val TruncateTransform = """^truncate\[(\d+)\]$""".r /** - * The names of the `truncate[W]` partition fields of `spec` whose source type is an integer - * (or a date) and whose width is above 2^30. + * The names of the `truncate[W]` partition fields of `spec` whose source type is an integer (or + * a date) and whose width is above 2^30. * * At those widths Iceberg Java's `TruncateUtil` overflows in `(v % W) + W`, while the format * spec and iceberg-rust compute `v - (v % W)`. A native write would therefore produce a diff --git a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala index b12fbb3f255..71c0e28b3c6 100644 --- a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala @@ -226,8 +226,8 @@ object CometIcebergTruncate * Largest integer `truncate` width whose `(v % W) + W` cannot overflow a Java `int`. * * Above it the result depends on the Iceberg jar the user runs: every release through 1.12.0 - * wraps, while a release carrying apache/iceberg#18112 returns the spec's `v - (v % W)`. - * Spark decides the expected answer, so those widths stay with Spark. + * wraps, while a release carrying apache/iceberg#18112 returns the spec's `v - (v % W)`. Spark + * decides the expected answer, so those widths stay with Spark. * https://github.com/apache/datafusion-comet/issues/7442 */ private val MaxSafeIntegerWidth: Int = 1 << 30 @@ -251,10 +251,9 @@ object CometIcebergTruncate CometIcebergSystemFunctions .positiveIntLiteral(parameter) .exists(_ > MaxSafeIntegerWidth) => - Unsupported( - Some( - s"truncate width above $MaxSafeIntegerWidth overflows Iceberg's int arithmetic, and " + - "the result depends on the Iceberg jar the user runs")) + Unsupported(Some( + s"truncate width above $MaxSafeIntegerWidth overflows Iceberg's int arithmetic, and " + + "the result depends on the Iceberg jar the user runs")) case _ => super.getSupportLevel(expr) } From c613a37c2fe69966ce25ab98c5aff4a16d060947 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 9 Nov 2026 17:58:32 -0300 Subject: [PATCH 4/5] test: keep truncate coverage inside the native width range --- .../apache/comet/CometIcebergSystemFunctionSuite.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index ec604000e21..e64b689a744 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -129,7 +129,8 @@ class CometIcebergSystemFunctionSuite withSourceTable { truncateColumns.foreach { column => val truncated = - Seq(1, 3, 10, 1000, Int.MaxValue).map(w => s"$catalog.system.truncate($w, $column)") + // Widths above 2^30 stay with Spark; the fallback has its own test below. + Seq(1, 3, 10, 1000, 1 << 30).map(w => s"$catalog.system.truncate($w, $column)") checkSparkAnswerAndOperator(s"SELECT $column, ${truncated.mkString(", ")} FROM $source") } } @@ -147,8 +148,8 @@ class CometIcebergSystemFunctionSuite "truncate width above") // Long, string and binary truncation does not overflow at any int width. checkSparkAnswerAndOperator( - s"SELECT $catalog.system.truncate(2000000000, i64), " + - s"$catalog.system.truncate(2000000000, str) FROM $source") + s"SELECT $catalog.system.truncate(${Int.MaxValue}, i64), " + + s"$catalog.system.truncate(${Int.MaxValue}, str) FROM $source") } } @@ -365,7 +366,7 @@ class CometIcebergSystemFunctionSuite .isInstanceOf[Unsupported]) // Long truncation uses long arithmetic, so the same width stays native. val truncateLong = - Class.forName("org.apache.iceberg.spark.functions.TruncateFunction$TruncateLong") + Class.forName("org.apache.iceberg.spark.functions.TruncateFunction$TruncateBigInt") val longValue = AttributeReference("l", LongType)() assert( level(CometIcebergTruncate, truncateLong, Literal((1 << 30) + 1), longValue) == From 964f29bfcdf4ced0a6f31d99f9d64c6c4706983d Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Mon, 9 Nov 2026 18:40:52 -0300 Subject: [PATCH 5/5] docs: state the wide truncate limit in the write eligibility table --- docs/source/user-guide/latest/iceberg.md | 2 +- .../scala/org/apache/comet/serde/icebergFunctions.scala | 8 +++++++- .../apache/comet/CometIcebergSystemFunctionSuite.scala | 3 +++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index 941a86efbb3..067b8e441ff 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -511,7 +511,7 @@ A write is eligible only when ALL of the following hold: | resolved `table.locationProvider()` | Iceberg's built-in `DefaultLocationProvider` | | Hadoop S3A settings for an `s3` / `s3a` data location | only `fs.s3a.access.key`, `secret.key`, `session.token`, `endpoint`, `endpoint.region`, and `path.style.access`, including their `fs.s3a.bucket..*` forms; any other effective `fs.s3a.*` setting falls back, apart from the ignored settings listed below | | Iceberg `FileIO` S3 settings for an `s3` / `s3a` data location | the S3 endpoint, region, static/session credentials, path-style, SSE (`none`, `s3`, `kms`, or `custom`; not `dsse-kms`), assume-role, anonymous/config-chain settings parsed by the pinned iceberg-rust version, plus Comet's credential-provider class and built-in web-identity properties. When a custom provider is configured, its vendor-owned `s3.*` / `client.*` properties are also forwarded; unsupported Iceberg-defined S3 settings still fall back | -| partition spec | any, except a `void` field whose source column was dropped beside a live field (see below) | +| partition spec | any, except a `void` field whose source column was dropped beside a live field (see below), and an integer `truncate[W]` field with `W` above 2^30 (see [Iceberg system functions](#iceberg-system-functions)) | | column types | any except `uuid` (Spark plans it as a string; no Arrow cast reaches `fixed(16)`) and the v3 types `variant`, `unknown`, `timestamp_ns`, `geometry` and `geography` | Within the namespaces that shape data-file bytes — `write.parquet.*` and `parquet.*` — diff --git a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala index 71c0e28b3c6..b58574a70e6 100644 --- a/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala +++ b/spark/src/main/scala/org/apache/comet/serde/icebergFunctions.scala @@ -261,7 +261,13 @@ object CometIcebergTruncate "Iceberg's `truncate(width, value)` system function on a `decimal` column. " + DecimalNote + " Truncating `-99999999999999.9999` in a `decimal(18,4)` column by a width of 10 is one " + "such value: the result has 19 digits. The other `truncate` input types, and `bucket` on " + - "decimals, are unaffected.") + "decimals, are unaffected.", + "Iceberg's `truncate(width, value)` system function on an `int`, `date`, `tinyint` or " + + "`smallint` column with a width above 1073741824. At those widths Iceberg Java's " + + "`(v % W) + W` overflows a 32-bit `int`, while the format spec and iceberg-rust compute " + + "`v - (v % W)`. Every Iceberg release through 1.12.0 overflows, so the expected answer " + + "depends on the Iceberg jar the user runs and Spark evaluates those calls. Long, string " + + "and binary truncation is unaffected.") } /** diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala index e64b689a744..5fbf94d1742 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergSystemFunctionSuite.scala @@ -396,6 +396,9 @@ class CometIcebergSystemFunctionSuite .getUnsupportedReasons() .exists(_.contains(CometIcebergTruncate.DecimalNote)), "the decimal truncate note is missing from CometStaticInvoke.getUnsupportedReasons") + assert( + CometStaticInvoke.getUnsupportedReasons().exists(_.contains("width above 1073741824")), + "the wide integer truncate note is missing from CometStaticInvoke.getUnsupportedReasons") // CometStaticInvoke dispatches on (functionName, class name), so the same expressions reach // the Iceberg handlers from there too.