From acf2cde65a23123eb3e08b3e52461fd958116ee9 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Tue, 10 Nov 2026 18:02:15 -0300 Subject: [PATCH 1/8] test(delta): add InterleaveBits handoff baseline --- .../delta/DeltaInterleaveBitsChecks.scala | 148 ++++++++++++++++++ dev/repros/delta/README-7573.md | 68 ++++++++ 2 files changed, 216 insertions(+) create mode 100644 dev/repros/delta/DeltaInterleaveBitsChecks.scala create mode 100644 dev/repros/delta/README-7573.md diff --git a/dev/repros/delta/DeltaInterleaveBitsChecks.scala b/dev/repros/delta/DeltaInterleaveBitsChecks.scala new file mode 100644 index 00000000000..eb830223b8b --- /dev/null +++ b/dev/repros/delta/DeltaInterleaveBitsChecks.scala @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet.repro + + +import java.nio.charset.StandardCharsets.UTF_8 +import java.nio.file.{Files, Paths} +import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Literal} +import org.apache.spark.sql.classic.ClassicConversions._ + +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.types._ + +/** Selected reference checks for #7573; no expression implementation is added. */ +object DeltaInterleaveBitsChecks { + def main(args: Array[String]): Unit = { + require(args.length == 2, "usage: stock|comet ") + val mode = args(0); require(Set("stock", "comet")(mode)) + val out = Paths.get(args(1)); Files.createDirectory(out) + val spark = SparkSession.builder + .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension" + + (if (mode == "comet") ",org.apache.comet.CometSparkSessionExtensions" else "")) + .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") + .config("spark.comet.enabled", (mode == "comet").toString) + .config("spark.comet.exec.enabled", (mode == "comet").toString) + .config("spark.comet.explain.fallback.enabled", "true") + .config("spark.shuffle.manager", if (mode == "comet") + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" + else "org.apache.spark.shuffle.sort.SortShuffleManager") + .config("spark.memory.offHeap.enabled", "true").config("spark.memory.offHeap.size", "512m") + .config("spark.sql.adaptive.enabled", "false").config("spark.sql.shuffle.partitions", "2") + .getOrCreate() + import spark.implicits._ + var passed = 0 + def check(name: String)(body: => Unit): Unit = { + try { body; Files.write(out.resolve(name + ".passed"), Array.emptyByteArray); passed += 1 } + catch { case e: Throwable => Files.write(out.resolve(name + ".error.txt"), e.toString.getBytes(UTF_8)); throw e } + } + def query(name: String, df: DataFrame)(assertRows: Array[Row] => Unit): Unit = check(name) { + val rows = df.collect(); assertRows(rows) + Files.write(out.resolve(name + ".rows.json"), rows.map(_.json).sorted.mkString("[", ",", "]").getBytes(UTF_8)) + Files.write(out.resolve(name + ".schema.json"), df.schema.json.getBytes(UTF_8)) + Files.write(out.resolve(name + ".plan.txt"), df.queryExecution.executedPlan.toString.getBytes(UTF_8)) + } + try { + val control = spark.range(20).selectExpr("sum(id) total") + query("control", control) { rows => require(rows.length == 1 && rows.head.getLong(0) == 190L) + if (mode == "comet") require(control.queryExecution.executedPlan.toString.contains("Comet")) } + def oracle(values: Seq[java.lang.Integer]): Array[Byte] = { + val bytes = new Array[Byte](4 * values.size); var at = 0 + for (bit <- 31 to 0 by -1; v <- values) { val n = if (v == null) 0 else v.intValue() + if (((n >>> bit) & 1) != 0) bytes(at / 8) = (bytes(at / 8) | (1 << (7-at%8))).toByte + at += 1 } + bytes + } + def hexBytes(b: Array[Byte]): String = b.map(x => f"${x & 255}%02X").mkString + val x = 0x000ff0ff; val y = 0xfff00f00 + check("interleave_asymmetric_oracle") { + require(hexBytes(oracle(Seq(x,y))) == "555555AAAA55AAAA") + require(!oracle(Seq(x,y)).sameElements(oracle(Seq(y,x)))) + require(hexBytes(oracle(Seq(-1,null))) == "AAAAAAAAAAAAAAAA") + require(hexBytes(oracle(Seq(null,-1))) == "5555555555555555") + require(!InterleaveBits(Seq(Literal("x"))).checkInputDataTypes().isSuccess) + } + val random = new scala.util.Random(7573L) + val pairs = Seq[(java.lang.Integer,java.lang.Integer)]((x,y),(y,x),(null,-1),(-1,null),(Int.MinValue,Int.MaxValue),(0,0)) ++ + (0 until 256).map(i => (if (i % 17 == 0) null else Integer.valueOf(random.nextInt()), if (i % 19 == 0) null else Integer.valueOf(random.nextInt()))) + val pairSchema = StructType(Seq(StructField("x",IntegerType),StructField("y",IntegerType))) + val pairFrame = spark.createDataFrame(spark.sparkContext.parallelize(pairs.map {case (a,b) => Row(a,b)},2),pairSchema) + val pairPath = out.resolve("pairs").toString; pairFrame.write.parquet(pairPath) + val inputs = spark.read.parquet(pairPath) + val expectedPairCounts = pairs.groupBy(p => p).map { case (pair, values) => pair -> values.size } + def assertInputPairs(rows: Array[Row]): Unit = { + val actual = rows.toSeq.map(r => ( + if (r.isNullAt(0)) null else Integer.valueOf(r.getInt(0)), + if (r.isNullAt(1)) null else Integer.valueOf(r.getInt(1)))) + require(actual.groupBy(p => p).map { case (pair, values) => pair -> values.size } == expectedPairCounts, + "input pairs changed, duplicated or disappeared") + } + def checkMixedRows(rows: Array[Row], literalFirst: Boolean): Unit = { + require(rows.length == pairs.size, "mixed expression output cardinality changed") + assertInputPairs(rows) + rows.foreach { r => + val value = if (literalFirst) {if (r.isNullAt(1)) null else Integer.valueOf(r.getInt(1))} else {if (r.isNullAt(0)) null else Integer.valueOf(r.getInt(0))} + require(r.getAs[Array[Byte]](2).sameElements(oracle(if (literalFirst) Seq(Integer.valueOf(-1),value) else Seq(value,Integer.valueOf(-1))))) } + } + check("interleave_mixed_validator_rejects_incomplete") { + for (bad <- Seq(Array.empty[Row],Array(Row(x,y,oracle(Seq(Integer.valueOf(x),Integer.valueOf(-1))))), Array.fill(pairs.size)(Row(x,y,oracle(Seq(Integer.valueOf(x),Integer.valueOf(-1))))))) { + var rejected = false + try checkMixedRows(bad,false) catch { case _: IllegalArgumentException => rejected = true } + assert(rejected,"mixed validator accepted empty, truncated or duplicated output") + } + } + for (fast <- Seq(false,true)) { + spark.conf.set(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key,fast.toString) + for (dims <- Seq(1,2)) { + val key = Column(InterleaveBits(Seq("x","y").take(dims).map(UnresolvedAttribute(_)))).as("key") + val df = inputs.select(col("x"),col("y"),key) + require(df.schema("key").dataType == BinaryType && !df.schema("key").nullable) + query(s"interleave_columns_${dims}_$fast",df)(rows => {require(rows.length == pairs.size);assertInputPairs(rows);rows.foreach { r => + val values = (0 until dims).map(i => if (r.isNullAt(i)) null else Integer.valueOf(r.getInt(i))) + require(r.getAs[Array[Byte]](2).sameElements(oracle(values))) }}) + } + for (literalFirst <- Seq(false,true)) { + val children = if (literalFirst) Seq(Literal(-1),UnresolvedAttribute("y")) else Seq(UnresolvedAttribute("x"),Literal(-1)) + val df = inputs.select(col("x"),col("y"),Column(InterleaveBits(children)).as("key")) + require(df.schema("key").dataType == BinaryType && !df.schema("key").nullable) + query(s"interleave_mixed_${literalFirst}_$fast",df)(rows => checkMixedRows(rows,literalFirst)) + } + spark.conf.set("spark.sql.optimizer.excludedRules","org.apache.spark.sql.catalyst.optimizer.ConstantFolding") + for (dims <- Seq(1,2)) { + val literal = inputs.limit(1).select(Column(InterleaveBits(Seq(x,y).take(dims).map(Literal(_)))).as("key")) + require(literal.queryExecution.optimizedPlan.exists(p => p.expressions.exists(_.exists(_.isInstanceOf[InterleaveBits]))),"literal expression folded despite exclusion") + require(literal.schema("key").dataType == BinaryType && !literal.schema("key").nullable) + query(s"interleave_literals_unfolded_${dims}_$fast",literal)(rows => { + require(rows.length == 1) + require(rows.head.getAs[Array[Byte]](0).sameElements(oracle(Seq(x,y).take(dims).map(Integer.valueOf)))) + }) + } + spark.conf.unset("spark.sql.optimizer.excludedRules") + } + require(passed == 15, s"expected 15 named cases, observed $passed") + Files.write(out.resolve("summary.json"),s"{\"mode\":\"$mode\",\"passed\":$passed,\"native_acceptance\":false}".getBytes(UTF_8)) + println(s"PASS DeltaInterleaveBitsChecks $mode cases=$passed (reference/fallback only)") + } finally spark.stop() + } +} diff --git a/dev/repros/delta/README-7573.md b/dev/repros/delta/README-7573.md new file mode 100644 index 00000000000..0bcacb7904d --- /dev/null +++ b/dev/repros/delta/README-7573.md @@ -0,0 +1,68 @@ + + +# Reference checks for #7573 + +Run commands from `dev/repros/delta`. + +Standalone, test-only probes related to [apache/datafusion-comet#7573](https://github.com/apache/datafusion-comet/issues/7573). They do not implement the requested Comet expression or change admission rules. The issue remains open. + +Run with Spark 4.2.0, Scala 2.13.18 and Java 17. Supply matching Delta Spark and storage JARs, version 4.4.0 or 4.4.1. Use a fresh output directory for every invocation. + +The setup requires `curl`. Set `SPARK_HOME` and `JAVA_HOME` to your local installations before running it. The Comet artifact is `org.apache.datafusion:comet-spark-spark4.2_2.13:1.2.0`; match the Spark 4.2 and Scala 2.13 lines. This assembly contains the platform library used by the validated Linux x86_64 run; no separate Comet companion JAR was used. Other platforms remain unverified. + +```bash +set -eu +: "${SPARK_HOME:?Set SPARK_HOME to a Spark 4.2.0 installation}" +: "${JAVA_HOME:?Set JAVA_HOME to a Java 17 installation}" +export PATH="$JAVA_HOME/bin:$PATH" +DELTA_VERSION=4.4.0 # Use 4.4.1 to repeat the other validated pairing. +ARTIFACT_DIR=$(mktemp -d) +export DELTA_SPARK_JAR="$ARTIFACT_DIR/delta-spark_2.13-$DELTA_VERSION.jar" +export DELTA_STORAGE_JAR="$ARTIFACT_DIR/delta-storage-$DELTA_VERSION.jar" +export COMET_JAR="$ARTIFACT_DIR/comet-spark-spark4.2_2.13-1.2.0.jar" +curl --fail --location --retry 3 -o "$DELTA_SPARK_JAR" \ + "https://repo.maven.apache.org/maven2/io/delta/delta-spark_2.13/$DELTA_VERSION/delta-spark_2.13-$DELTA_VERSION.jar" +curl --fail --location --retry 3 -o "$DELTA_STORAGE_JAR" \ + "https://repo.maven.apache.org/maven2/io/delta/delta-storage/$DELTA_VERSION/delta-storage-$DELTA_VERSION.jar" +curl --fail --location --retry 3 -o "$COMET_JAR" \ + https://repo.maven.apache.org/maven2/org/apache/datafusion/comet-spark-spark4.2_2.13/1.2.0/comet-spark-spark4.2_2.13-1.2.0.jar +mkdir classes +java -Xmx1g -cp "$SPARK_HOME/jars/*" scala.tools.nsc.Main \ + -classpath "$SPARK_HOME/jars/*:$DELTA_SPARK_JAR:$DELTA_STORAGE_JAR" \ + -d classes DeltaInterleaveBitsChecks.scala +jar cf checks.jar -C classes . +"$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ + --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR" \ + --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ + checks.jar stock /tmp/DeltaInterleaveBitsChecks-stock +"$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ + --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR,$COMET_JAR" \ + --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ + checks.jar comet /tmp/DeltaInterleaveBitsChecks-comet +``` + +Expected: 15 named cases, matching row/schema JSON across stock and Comet runs, and `native_acceptance: false` in `summary.json`. The Comet lane requires an accelerated control query; passing Delta probes can still execute in Spark. Execution plans are saved beside rows. These are manually run probes, not tests wired into the Comet CI reactor. + +Grounding: selected equivalents of the following versioned upstream tests, not a replacement for their full suites: + +- [InterleaveBitsSuite (v4.4.0)](https://github.com/delta-io/delta/blob/v4.4.0/spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsSuite.scala) +- [InterleaveBitsSuite (v4.4.1)](https://github.com/delta-io/delta/blob/v4.4.1/spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsSuite.scala) + +Validation uses Comet 1.2.0, not a build of this branch. Current-main expression support and the production image remain unverified. From cf533e38b4e9eb93701f59bf82e282e7239d6147 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Tue, 10 Nov 2026 18:34:25 -0300 Subject: [PATCH 2/8] docs(delta): isolate each InterleaveBits run --- dev/repros/delta/README-7573.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/dev/repros/delta/README-7573.md b/dev/repros/delta/README-7573.md index 0bcacb7904d..7d0ae70ab07 100644 --- a/dev/repros/delta/README-7573.md +++ b/dev/repros/delta/README-7573.md @@ -33,7 +33,9 @@ set -eu : "${JAVA_HOME:?Set JAVA_HOME to a Java 17 installation}" export PATH="$JAVA_HOME/bin:$PATH" DELTA_VERSION=4.4.0 # Use 4.4.1 to repeat the other validated pairing. -ARTIFACT_DIR=$(mktemp -d) +RUN_DIR=$(mktemp -d /tmp/comet-interleave.XXXXXX) +ARTIFACT_DIR="$RUN_DIR/artifacts" +mkdir "$ARTIFACT_DIR" export DELTA_SPARK_JAR="$ARTIFACT_DIR/delta-spark_2.13-$DELTA_VERSION.jar" export DELTA_STORAGE_JAR="$ARTIFACT_DIR/delta-storage-$DELTA_VERSION.jar" export COMET_JAR="$ARTIFACT_DIR/comet-spark-spark4.2_2.13-1.2.0.jar" @@ -43,19 +45,19 @@ curl --fail --location --retry 3 -o "$DELTA_STORAGE_JAR" \ "https://repo.maven.apache.org/maven2/io/delta/delta-storage/$DELTA_VERSION/delta-storage-$DELTA_VERSION.jar" curl --fail --location --retry 3 -o "$COMET_JAR" \ https://repo.maven.apache.org/maven2/org/apache/datafusion/comet-spark-spark4.2_2.13/1.2.0/comet-spark-spark4.2_2.13-1.2.0.jar -mkdir classes +mkdir "$RUN_DIR/classes" java -Xmx1g -cp "$SPARK_HOME/jars/*" scala.tools.nsc.Main \ -classpath "$SPARK_HOME/jars/*:$DELTA_SPARK_JAR:$DELTA_STORAGE_JAR" \ - -d classes DeltaInterleaveBitsChecks.scala -jar cf checks.jar -C classes . + -d "$RUN_DIR/classes" DeltaInterleaveBitsChecks.scala +jar cf "$RUN_DIR/checks.jar" -C "$RUN_DIR/classes" . "$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR" \ --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ - checks.jar stock /tmp/DeltaInterleaveBitsChecks-stock + "$RUN_DIR/checks.jar" stock "$RUN_DIR/stock" "$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR,$COMET_JAR" \ --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ - checks.jar comet /tmp/DeltaInterleaveBitsChecks-comet + "$RUN_DIR/checks.jar" comet "$RUN_DIR/comet" ``` Expected: 15 named cases, matching row/schema JSON across stock and Comet runs, and `native_acceptance: false` in `summary.json`. The Comet lane requires an accelerated control query; passing Delta probes can still execute in Spark. Execution plans are saved beside rows. These are manually run probes, not tests wired into the Comet CI reactor. From 8ae7768a5056d3745fb521e208976311537bb1e7 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Wed, 11 Nov 2026 19:49:50 +0700 Subject: [PATCH 3/8] feat: add a native kernel for Delta's InterleaveBits Adds delta_interleave_bits, a scalar function that builds Delta Lake's Z-order key for one or two Int32 inputs: bits are taken from the most significant down, alternating between inputs with the first input in the higher position, a null input counts as zero, and the result is four bytes per input and never null. The tests check the output against values produced by Delta's own InterleaveBits on both its default and fast code paths, including signed extrema and nulls. Nothing calls the function from Spark yet; the Delta serde that uses it comes separately. --- native/spark-expr/src/comet_scalar_funcs.rs | 9 +- .../src/delta_funcs/interleave_bits.rs | 421 ++++++++++++++++++ native/spark-expr/src/delta_funcs/mod.rs | 22 + native/spark-expr/src/lib.rs | 2 + 4 files changed, 450 insertions(+), 4 deletions(-) create mode 100644 native/spark-expr/src/delta_funcs/interleave_bits.rs create mode 100644 native/spark-expr/src/delta_funcs/mod.rs diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 0086538b276..e7f968166be 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -32,10 +32,10 @@ use crate::{ spark_ceil, spark_day_name, spark_decimal_div, spark_decimal_integral_div, spark_floor, spark_isnan, spark_lpad, spark_make_decimal, spark_month_name, spark_read_side_padding, spark_round, spark_rpad, spark_sequence, spark_sequence_recomputed, spark_to_time, spark_unhex, - spark_unscaled_value, EvalMode, SparkArrayExtrema, SparkArrayPositionFunc, SparkArrayRemove, - SparkArraySetOp, SparkArraySlice, SparkArraySort, SparkArraysOverlap, SparkContains, - SparkDateDiff, SparkDateFromUnixDate, SparkDateTrunc, SparkDayOfWeek, SparkFlatten, - SparkFloatArrayContains, SparkIcebergBucket, SparkIcebergTemporalTransform, + spark_unscaled_value, DeltaInterleaveBits, EvalMode, SparkArrayExtrema, SparkArrayPositionFunc, + SparkArrayRemove, SparkArraySetOp, SparkArraySlice, SparkArraySort, SparkArraysOverlap, + SparkContains, SparkDateDiff, SparkDateFromUnixDate, SparkDateTrunc, SparkDayOfWeek, + SparkFlatten, SparkFloatArrayContains, SparkIcebergBucket, SparkIcebergTemporalTransform, SparkIcebergTruncate, SparkMakeDate, SparkMakeInterval, SparkMakeTime, SparkMapExtract, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, SparkSortArray, SparkWeekDay, }; @@ -365,6 +365,7 @@ fn all_scalar_functions() -> Vec> { Arc::new(ScalarUDF::new_from_impl(SparkDateTrunc::default())), Arc::new(ScalarUDF::new_from_impl(SparkDayOfWeek::default())), Arc::new(ScalarUDF::new_from_impl(SparkWeekDay::default())), + Arc::new(ScalarUDF::new_from_impl(DeltaInterleaveBits::default())), Arc::new(ScalarUDF::new_from_impl(SparkFlatten::default())), Arc::new(ScalarUDF::new_from_impl(SparkIcebergBucket::default())), Arc::new(ScalarUDF::new_from_impl(SparkIcebergTruncate::default())), diff --git a/native/spark-expr/src/delta_funcs/interleave_bits.rs b/native/spark-expr/src/delta_funcs/interleave_bits.rs new file mode 100644 index 00000000000..3480a13a249 --- /dev/null +++ b/native/spark-expr/src/delta_funcs/interleave_bits.rs @@ -0,0 +1,421 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::{Array, AsArray, BinaryArray, BinaryBuilder, Int32Array}; +use arrow::datatypes::{DataType, Int32Type}; +use datafusion::common::{plan_err, Result, ScalarValue}; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, +}; +use std::sync::Arc; + +const NAME: &str = "delta_interleave_bits"; + +/// Largest number of inputs the kernel accepts; two 32-bit inputs fill the `u64` key. +const MAX_INPUTS: usize = 2; + +const BITS_PER_INPUT: u32 = i32::BITS; +const BYTES_PER_INPUT: usize = 4; + +/// Delta Lake's `InterleaveBits`, the Z-order key `OPTIMIZE ... ZORDER BY` sorts on. +/// +/// Takes one or two `Int32` inputs and returns `4 * n` bytes: bits 31 down to 0 of each input's +/// two's-complement value, round-robin in argument order, so the first input supplies the higher +/// bit of each group. A null input counts as 0 and the result is never null. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct DeltaInterleaveBits { + signature: Signature, +} + +impl DeltaInterleaveBits { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Int32]), + TypeSignature::Exact(vec![DataType::Int32, DataType::Int32]), + ], + Volatility::Immutable, + ), + } + } +} + +impl Default for DeltaInterleaveBits { + fn default() -> Self { + Self::new() + } +} + +impl ScalarUDFImpl for DeltaInterleaveBits { + fn name(&self) -> &str { + NAME + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + check_arg_types(arg_types)?; + Ok(DataType::Binary) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let number_rows = args.number_rows; + let args = args.args; + check_arg_types(&args.iter().map(|a| a.data_type()).collect::>())?; + let all_scalar = args.iter().all(|a| matches!(a, ColumnarValue::Scalar(_))); + let rows = if all_scalar { 1 } else { number_rows }; + let arrays = args + .iter() + .map(|a| a.to_array(rows)) + .collect::>>()?; + let columns = arrays + .iter() + .map(|a| a.as_primitive::()) + .collect::>(); + let result = interleave_columns(&columns, rows); + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(Arc::new(result))) + } + } +} + +fn check_arg_types(arg_types: &[DataType]) -> Result<()> { + if !(1..=MAX_INPUTS).contains(&arg_types.len()) { + return plan_err!( + "{NAME} expects 1 to {MAX_INPUTS} arguments, got {}", + arg_types.len() + ); + } + match arg_types.iter().find(|t| **t != DataType::Int32) { + Some(other) => plan_err!("{NAME} expects Int32 input, got {other}"), + None => Ok(()), + } +} + +fn interleave_columns(columns: &[&Int32Array], rows: usize) -> BinaryArray { + let width = BYTES_PER_INPUT * columns.len(); + let mut builder = BinaryBuilder::with_capacity(rows, rows * width); + let mut values = [0u32; MAX_INPUTS]; + for row in 0..rows { + for (value, column) in values.iter_mut().zip(columns) { + *value = if column.is_null(row) { + 0 + } else { + column.value(row) as u32 + }; + } + let key = interleave(&values[..columns.len()]); + builder.append_value(&key.to_be_bytes()[size_of::() - width..]); + } + builder.finish() +} + +/// Interleaves `values` MSB-first, one bit from each value in turn, into the low +/// `32 * values.len()` bits of the key. +fn interleave(values: &[u32]) -> u64 { + let mut key = 0u64; + for pos in (0..BITS_PER_INPUT).rev() { + for &value in values { + key = (key << 1) | u64::from((value >> pos) & 1); + } + } + key +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::ArrayRef; + use arrow::datatypes::Field; + use datafusion::config::ConfigOptions; + + /// Interleaved keys Delta's own `InterleaveBits.interleaveBits` produced for these inputs, + /// with the fast path both on and off (the two agreed on every row). + const DELTA_GOLDENS: &[(&[i32], &str)] = &[ + (&[i32::MIN], "80000000"), + (&[i32::MAX], "7FFFFFFF"), + (&[0], "00000000"), + (&[-1], "FFFFFFFF"), + (&[1], "00000001"), + (&[1431655765], "55555555"), + (&[-1431655766], "AAAAAAAA"), + (&[1044735], "000FF0FF"), + (&[-1044736], "FFF00F00"), + (&[16909060], "01020304"), + (&[i32::MIN, i32::MIN], "C000000000000000"), + (&[i32::MIN, i32::MAX], "9555555555555555"), + (&[i32::MIN, 0], "8000000000000000"), + (&[i32::MIN, -1], "D555555555555555"), + (&[i32::MIN, 1], "8000000000000001"), + (&[i32::MIN, 1431655765], "9111111111111111"), + (&[i32::MIN, -1431655766], "C444444444444444"), + (&[i32::MIN, 1044735], "8000005555005555"), + (&[i32::MIN, -1044736], "D555550000550000"), + (&[i32::MIN, 16909060], "8001000400050010"), + (&[i32::MAX, i32::MIN], "6AAAAAAAAAAAAAAA"), + (&[i32::MAX, i32::MAX], "3FFFFFFFFFFFFFFF"), + (&[i32::MAX, 0], "2AAAAAAAAAAAAAAA"), + (&[i32::MAX, -1], "7FFFFFFFFFFFFFFF"), + (&[i32::MAX, 1], "2AAAAAAAAAAAAAAB"), + (&[i32::MAX, 1431655765], "3BBBBBBBBBBBBBBB"), + (&[i32::MAX, -1431655766], "6EEEEEEEEEEEEEEE"), + (&[i32::MAX, 1044735], "2AAAAAFFFFAAFFFF"), + (&[i32::MAX, -1044736], "7FFFFFAAAAFFAAAA"), + (&[i32::MAX, 16909060], "2AABAAAEAAAFAABA"), + (&[0, i32::MIN], "4000000000000000"), + (&[0, i32::MAX], "1555555555555555"), + (&[0, 0], "0000000000000000"), + (&[0, -1], "5555555555555555"), + (&[0, 1], "0000000000000001"), + (&[0, 1431655765], "1111111111111111"), + (&[0, -1431655766], "4444444444444444"), + (&[0, 1044735], "0000005555005555"), + (&[0, -1044736], "5555550000550000"), + (&[0, 16909060], "0001000400050010"), + (&[-1, i32::MIN], "EAAAAAAAAAAAAAAA"), + (&[-1, i32::MAX], "BFFFFFFFFFFFFFFF"), + (&[-1, 0], "AAAAAAAAAAAAAAAA"), + (&[-1, -1], "FFFFFFFFFFFFFFFF"), + (&[-1, 1], "AAAAAAAAAAAAAAAB"), + (&[-1, 1431655765], "BBBBBBBBBBBBBBBB"), + (&[-1, -1431655766], "EEEEEEEEEEEEEEEE"), + (&[-1, 1044735], "AAAAAAFFFFAAFFFF"), + (&[-1, -1044736], "FFFFFFAAAAFFAAAA"), + (&[-1, 16909060], "AAABAAAEAAAFAABA"), + (&[1, i32::MIN], "4000000000000002"), + (&[1, i32::MAX], "1555555555555557"), + (&[1, 0], "0000000000000002"), + (&[1, -1], "5555555555555557"), + (&[1, 1], "0000000000000003"), + (&[1, 1431655765], "1111111111111113"), + (&[1, -1431655766], "4444444444444446"), + (&[1, 1044735], "0000005555005557"), + (&[1, -1044736], "5555550000550002"), + (&[1, 16909060], "0001000400050012"), + (&[1431655765, i32::MIN], "6222222222222222"), + (&[1431655765, i32::MAX], "3777777777777777"), + (&[1431655765, 0], "2222222222222222"), + (&[1431655765, -1], "7777777777777777"), + (&[1431655765, 1], "2222222222222223"), + (&[1431655765, 1431655765], "3333333333333333"), + (&[1431655765, -1431655766], "6666666666666666"), + (&[1431655765, 1044735], "2222227777227777"), + (&[1431655765, -1044736], "7777772222772222"), + (&[1431655765, 16909060], "2223222622272232"), + (&[-1431655766, i32::MIN], "C888888888888888"), + (&[-1431655766, i32::MAX], "9DDDDDDDDDDDDDDD"), + (&[-1431655766, 0], "8888888888888888"), + (&[-1431655766, -1], "DDDDDDDDDDDDDDDD"), + (&[-1431655766, 1], "8888888888888889"), + (&[-1431655766, 1431655765], "9999999999999999"), + (&[-1431655766, -1431655766], "CCCCCCCCCCCCCCCC"), + (&[-1431655766, 1044735], "888888DDDD88DDDD"), + (&[-1431655766, -1044736], "DDDDDD8888DD8888"), + (&[-1431655766, 16909060], "8889888C888D8898"), + (&[1044735, i32::MIN], "400000AAAA00AAAA"), + (&[1044735, i32::MAX], "155555FFFF55FFFF"), + (&[1044735, 0], "000000AAAA00AAAA"), + (&[1044735, -1], "555555FFFF55FFFF"), + (&[1044735, 1], "000000AAAA00AAAB"), + (&[1044735, 1431655765], "111111BBBB11BBBB"), + (&[1044735, -1431655766], "444444EEEE44EEEE"), + (&[1044735, 1044735], "000000FFFF00FFFF"), + (&[1044735, -1044736], "555555AAAA55AAAA"), + (&[1044735, 16909060], "000100AEAA05AABA"), + (&[-1044736, i32::MIN], "EAAAAA0000AA0000"), + (&[-1044736, i32::MAX], "BFFFFF5555FF5555"), + (&[-1044736, 0], "AAAAAA0000AA0000"), + (&[-1044736, -1], "FFFFFF5555FF5555"), + (&[-1044736, 1], "AAAAAA0000AA0001"), + (&[-1044736, 1431655765], "BBBBBB1111BB1111"), + (&[-1044736, -1431655766], "EEEEEE4444EE4444"), + (&[-1044736, 1044735], "AAAAAA5555AA5555"), + (&[-1044736, -1044736], "FFFFFF0000FF0000"), + (&[-1044736, 16909060], "AAABAA0400AF0010"), + (&[16909060, i32::MIN], "40020008000A0020"), + (&[16909060, i32::MAX], "1557555D555F5575"), + (&[16909060, 0], "00020008000A0020"), + (&[16909060, -1], "5557555D555F5575"), + (&[16909060, 1], "00020008000A0021"), + (&[16909060, 1431655765], "11131119111B1131"), + (&[16909060, -1431655766], "4446444C444E4464"), + (&[16909060, 1044735], "0002005D550A5575"), + (&[16909060, -1044736], "55575508005F0020"), + (&[16909060, 16909060], "0003000C000F0030"), + (&[-194441980, -713859073], "FB313D8711F75575"), + (&[1465097161, 551116650], "262A734B93DFB4C6"), + (&[-2044496387, -1402509360], "C4781C1F3E37FBA2"), + (&[-42021569, 936478219], "AFB77BA9E0B60AEF"), + (&[-371001683, -1494708188], "EC96FC48EB288CB2"), + (&[-11981971, -1746055248], "EBBF74D318DF6DA2"), + (&[1396948856, -1908031548], "625E301BCFFF7A90"), + (&[-964581762, -717158876], "F1399003883C2EB8"), + (&[-1802157460, 629602743], "8631C23675496DB5"), + (&[47397773, -796979038], "5108B75F0B8AC4A6"), + (&[2119901077, 859130386], "2FAD279B12EE8326"), + (&[-681693820, -2140680892], "E22A36BD58F39030"), + (&[2129441368, -1087931688], "6FFDACB59F4C73C0"), + (&[458445216, -393048690], "56CA630E6253C854"), + (&[1762056806, -117967399], "7DC2553DF7997969"), + (&[1232985102, -1191240484], "65C27FF7A3F051F8"), + (&[-650702380, -866216185], "F2D21B7E4317A235"), + (&[-1061812808, 807767500], "A5008E394073DAD0"), + (&[-293689653, 887702561], "ADB87EE99838A48B"), + (&[1552189062, -1186747436], "67E19025C46DD138"), + ]; + + fn invoke(args: Vec, number_rows: usize) -> Result { + DeltaInterleaveBits::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows, + return_field: Arc::new(Field::new("result", DataType::Binary, false)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn int_array(values: Vec>) -> ColumnarValue { + ColumnarValue::Array(Arc::new(Int32Array::from(values))) + } + + /// Invokes the function on array arguments and returns each row as upper-case hex. + fn hex_rows(columns: Vec>>) -> Vec { + let rows = columns[0].len(); + let args = columns.into_iter().map(int_array).collect(); + let ColumnarValue::Array(result) = invoke(args, rows).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(result.null_count(), 0); + to_hex_rows(&result) + } + + fn to_hex_rows(array: &ArrayRef) -> Vec { + array + .as_binary::() + .iter() + .map(|bytes| to_hex(bytes.expect("result is never null"))) + .collect() + } + + fn to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02X}")).collect() + } + + #[test] + fn oracle_two_inputs() { + let rows = hex_rows(vec![ + vec![Some(0x000ff0ff)], + vec![Some(0xfff00f00_u32 as i32)], + ]); + assert_eq!(rows, ["555555AAAA55AAAA"]); + } + + #[test] + fn null_counts_as_zero() { + let rows = hex_rows(vec![vec![Some(-1), None], vec![None, Some(-1)]]); + assert_eq!(rows, ["AAAAAAAAAAAAAAAA", "5555555555555555"]); + } + + #[test] + fn one_input_is_big_endian() { + let rows = hex_rows(vec![vec![Some(i32::MIN), Some(-1), Some(0x01020304)]]); + assert_eq!(rows, ["80000000", "FFFFFFFF", "01020304"]); + } + + #[test] + fn scalar_broadcasts() { + let column = vec![Some(1), Some(2), None]; + let minus_one = || ColumnarValue::Scalar(ScalarValue::Int32(Some(-1))); + let cases = [ + ( + vec![int_array(column.clone()), minus_one()], + vec![column.clone(), vec![Some(-1); 3]], + ), + ( + vec![minus_one(), int_array(column.clone())], + vec![vec![Some(-1); 3], column.clone()], + ), + ]; + for (args, expanded) in cases { + let ColumnarValue::Array(result) = invoke(args, column.len()).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(to_hex_rows(&result), hex_rows(expanded)); + } + } + + #[test] + fn all_scalar_args() { + let cases: [Vec>; 5] = [ + vec![Some(0x000ff0ff), Some(0xfff00f00_u32 as i32)], + vec![None, Some(-1)], + vec![None, None], + vec![Some(0x01020304)], + vec![None], + ]; + for values in cases { + let args = values + .iter() + .map(|v| ColumnarValue::Scalar(ScalarValue::Int32(*v))) + .collect(); + let ColumnarValue::Scalar(ScalarValue::Binary(Some(bytes))) = invoke(args, 1).unwrap() + else { + panic!("expected a non-null binary scalar for {values:?}"); + }; + let expected = hex_rows(values.iter().map(|v| vec![*v]).collect()); + assert_eq!(vec![to_hex(&bytes)], expected, "{values:?}"); + } + } + + #[test] + fn delta_goldens() { + for (inputs, expected) in DELTA_GOLDENS { + let columns = inputs.iter().map(|v| vec![Some(*v)]).collect(); + assert_eq!(hex_rows(columns), [*expected], "{inputs:?}"); + } + } + + #[test] + fn rejects_bad_arity() { + let udf = DeltaInterleaveBits::new(); + for arity in [0, 3] { + assert!(udf.return_type(&vec![DataType::Int32; arity]).is_err()); + let args = vec![ColumnarValue::Scalar(ScalarValue::Int32(Some(1))); arity]; + assert!(invoke(args, 1).is_err()); + } + } + + #[test] + fn rejects_non_int32_input() { + let udf = DeltaInterleaveBits::new(); + for arg_types in [ + vec![DataType::Int64], + vec![DataType::Int32, DataType::Int64], + ] { + assert!(udf.return_type(&arg_types).is_err(), "{arg_types:?}"); + } + let args = vec![ColumnarValue::Scalar(ScalarValue::Int64(Some(1)))]; + assert!(invoke(args, 1).is_err()); + } +} diff --git a/native/spark-expr/src/delta_funcs/mod.rs b/native/spark-expr/src/delta_funcs/mod.rs new file mode 100644 index 00000000000..f31b4aad5b5 --- /dev/null +++ b/native/spark-expr/src/delta_funcs/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native implementations of Delta Lake expressions. + +mod interleave_bits; + +pub use interleave_bits::DeltaInterleaveBits; diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs index ea8316bcce5..a557a38c664 100644 --- a/native/spark-expr/src/lib.rs +++ b/native/spark-expr/src/lib.rs @@ -48,8 +48,10 @@ pub mod hash_funcs; mod string_funcs; mod datetime_funcs; +mod delta_funcs; mod iceberg_funcs; pub use agg_funcs::*; +pub use delta_funcs::DeltaInterleaveBits; pub use iceberg_funcs::{SparkIcebergBucket, SparkIcebergTemporalTransform, SparkIcebergTruncate}; pub use cast::{spark_cast, Cast, SparkCastOptions}; From 3927f04161234ab6c760b5c8831bfcbcd4c8d1e9 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Wed, 11 Nov 2026 22:19:29 +0700 Subject: [PATCH 4/8] feat: let contribs register expression serdes Adds CometExpressionContrib, a ServiceLoader hook beside CometScanContrib, so a contrib can register a serde for an expression class core never names. Delta's InterleaveBits is the first user: the class lives in delta-spark, which core does not depend on. QueryPlanSerde.serdeFor now looks in core's map first and the contrib registry second, and both lookup sites use it. A contrib entry for a class core already handles is ignored, a class claimed by two contribs is dropped from both since ServiceLoader order is unspecified, and a contrib that throws is skipped. Default builds ship no service file, so nothing changes for them. The scan suite's service-file test helper moves into a shared test utility that both suites use. --- .github/workflows/pr_build_linux.yml | 1 + .github/workflows/pr_build_macos.yml | 1 + .../org/apache/comet/ContribServices.scala | 5 +- .../comet/serde/CometExpressionContrib.scala | 119 +++++++++++++ .../apache/comet/serde/CometScalaUDF.scala | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 21 ++- .../apache/comet/ContribServiceFiles.scala | 58 +++++++ .../comet/rules/CometScanContribSuite.scala | 36 +--- .../serde/CometExpressionContribSuite.scala | 157 ++++++++++++++++++ 9 files changed, 362 insertions(+), 38 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala create mode 100644 spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala create mode 100644 spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index b176bc829ad..1febe7a57f4 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -688,6 +688,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index a9b1a12832d..f539f28d417 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -326,6 +326,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/spark/src/main/scala/org/apache/comet/ContribServices.scala b/spark/src/main/scala/org/apache/comet/ContribServices.scala index 879ed2baac2..0a912d007bf 100644 --- a/spark/src/main/scala/org/apache/comet/ContribServices.scala +++ b/spark/src/main/scala/org/apache/comet/ContribServices.scala @@ -30,7 +30,7 @@ import org.apache.comet.util.ClassLoaders /** * Shared `ServiceLoader` discovery for Comet's optional, out-of-tree contrib registries - * (`CometScanContrib`, `CometConfigProvider`, `PlanDataInjector`). + * (`CometScanContrib`, `CometExpressionContrib`, `CometConfigProvider`, `PlanDataInjector`). * * Exists because the obvious spellings are both wrong in ways that are invisible until a contrib * jar is misbuilt in production: @@ -53,8 +53,7 @@ import org.apache.comet.util.ClassLoaders * changes. * * Not `private[comet]`: `PlanDataInjector` lives in `org.apache.spark.sql.comet`, which is a - * different package root, so a comet-scoped qualifier would hide this from one of its three call - * sites. + * different package root, so a comet-scoped qualifier would hide this from one of its call sites. */ object ContribServices extends Logging { diff --git a/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala new file mode 100644 index 00000000000..26ff4227fb7 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.util.control.NonFatal + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Expression + +import org.apache.comet.ContribServices + +/** + * Hook that lets an optional, out-of-tree contrib register serdes for expression classes core + * does not know about. Core names no contrib: implementations are discovered at runtime via the + * JDK [[java.util.ServiceLoader]], so default builds (which ship no service file) see an empty + * registry. + * + * A contrib ships an implementation of this trait and a + * `META-INF/services/org.apache.comet.serde.CometExpressionContrib` resource naming it. + * + * Core serdes always win: [[QueryPlanSerde.serdeFor]] consults core first, and a contrib entry + * for a class core registers is ignored. A class that two contribs both claim is dropped from + * both, because ServiceLoader order is not specified and so neither can be preferred. + * + * `expressionSerdes` runs while the registry is being built, so it must not call + * [[QueryPlanSerde.serdeFor]] or otherwise convert expressions. + */ +trait CometExpressionContrib { + + /** Serdes this contrib provides, keyed by the expression class each one converts. */ + def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] +} + +object CometExpressionContrib extends Logging { + + /** + * Contrib serdes for classes core does not register, discovered through the thread context + * ClassLoader (see `ContribServices.load`). Empty on a default build. + */ + private[comet] lazy val registered: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + merge( + QueryPlanSerde.exprSerdeMap.keySet.toSet[Class[_]], + ContribServices.load(classOf[CometExpressionContrib], getClass.getClassLoader)) + + /** Discover implementations visible to `loader`, without touching `registered`. */ + private[comet] def loadContribs(loader: ClassLoader): Seq[CometExpressionContrib] = + ContribServices.loadFrom(classOf[CometExpressionContrib], loader) + + /** + * Combine the serdes of `contribs`, dropping classes in `core` and classes more than one + * contrib claims. A contrib whose `expressionSerdes` throws is skipped with a warning, so one + * broken contrib jar neither fails planning nor hides the serdes of the others. As in + * `CometScanContrib`, a `LinkageError` (a contrib built against other Comet internals) is + * contained the same way, while fatal errors propagate. + */ + private[comet] def merge(core: Set[Class[_]], contribs: Seq[CometExpressionContrib]) + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = { + val claims = contribs.flatMap { contrib => + serdesOf(contrib).map { case (cls, serde) => Claim(cls, contrib.getClass.getName, serde) } + } + val kept = claims.groupBy(_.cls).values.toList.flatMap { entries => + val cls = entries.head.cls + val owners = entries.map(_.owner).mkString(", ") + if (core.contains(cls)) { + logWarning( + s"Ignoring the ${cls.getName} serde from $owners: Comet already has a serde for " + + "this expression.") + None + } else if (entries.size > 1) { + logWarning( + s"Ignoring every serde for ${cls.getName}: more than one contrib registers one " + + s"($owners), and their discovery order is not specified.") + None + } else { + entries.headOption + } + } + kept.map(claim => claim.cls -> claim.serde).toMap + } + + private case class Claim( + cls: Class[_ <: Expression], + owner: String, + serde: CometExpressionSerde[_]) + + private def serdesOf( + contrib: CometExpressionContrib): Seq[(Class[_ <: Expression], CometExpressionSerde[_])] = + try contrib.expressionSerdes.toList + catch { + case NonFatal(e) => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed", + e) + Nil + case e: LinkageError => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed with " + + s"${e.getClass.getName}, so it was likely built against a different Comet version", + e) + Nil + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 7244875e2cb..6596a69eacd 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -94,7 +94,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { */ private def isEnabled(guard: If): Boolean = { def disabled(e: Expression): Boolean = - QueryPlanSerde.exprSerdeMap.get(e.getClass).exists { serde => + QueryPlanSerde.serdeFor(e.getClass).exists { serde => !CometConf.isExprEnabled( serde.asInstanceOf[CometExpressionSerde[Expression]].getExprConfigName(e)) } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8b89b0742c..1f40a861ca1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -413,6 +413,25 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { val exprSerdeMap: Map[Class[_ <: Expression], CometExpressionSerde[_]] = serdeGroups.map(_._2).reduce(_ ++ _) + /** + * The serde for expression class `cls`: core's if it has one, otherwise one registered by a + * [[CometExpressionContrib]]. + */ + def serdeFor(cls: Class[_]): Option[CometExpressionSerde[_]] = + serdeFor(cls, CometExpressionContrib.registered) + + /** `serdeFor` against an explicit contrib registry, so tests need not load the real one. */ + private[comet] def serdeFor( + cls: Class[_], + contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]]) + : Option[CometExpressionSerde[_]] = { + // The maps' key type is invariant in its bound; a non-Expression class just misses. + val key = cls.asInstanceOf[Class[_ <: Expression]] + val core = exprSerdeMap.get(key) + // An explicit branch rather than `orElse`, which would allocate a closure on every lookup. + if (core.isDefined) core else contribSerdes.get(key) + } + /** * Mapping of Spark aggregate expression class to Comet expression handler. */ @@ -1120,7 +1139,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { exprToProtoInternal(child, inputs, binding) case expr => - QueryPlanSerde.exprSerdeMap.get(expr.getClass) match { + QueryPlanSerde.serdeFor(expr.getClass) match { case Some(handler) => convert(expr, handler.asInstanceOf[CometExpressionSerde[Expression]]) case _ => diff --git a/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala new file mode 100644 index 00000000000..40f0c7cf254 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File +import java.net.URLClassLoader +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +/** Test support for driving real `ServiceLoader` discovery of contrib registries. */ +object ContribServiceFiles { + + /** + * Writes a `META-INF/services/` file naming `implNames` into a temp directory and + * hands `f` an isolated child `URLClassLoader` that sees it. The child loader keeps the + * discovery out of the JVM-wide registries, which the default-build tests assert are empty. + */ + def withServiceFile(service: Class[_], implNames: Seq[String])(f: ClassLoader => Unit): Unit = { + // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; + // ensure it before createTempDirectory (which requires its parent to exist). + val baseTmp = new File(System.getProperty("java.io.tmpdir")) + baseTmp.mkdirs() + val svcDir = + Files.createTempDirectory(baseTmp.toPath, s"comet-${service.getSimpleName}-test").toFile + try { + val servicesDir = new File(svcDir, "META-INF/services") + assert(servicesDir.mkdirs(), s"could not create $servicesDir") + Files.write( + new File(servicesDir, service.getName).toPath, + implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) + + f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) + } finally { + def del(file: File): Unit = { + Option(file.listFiles()).foreach(_.foreach(del)) + file.delete() + } + del(svcDir) + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 4e41a9b147a..ffbe593591d 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -19,10 +19,6 @@ package org.apache.comet.rules -import java.io.File -import java.net.URLClassLoader -import java.nio.charset.StandardCharsets -import java.nio.file.Files import java.util.ServiceLoader import scala.annotation.nowarn @@ -43,7 +39,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, LeafExecNode, SparkPl import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.comet.ContribServices +import org.apache.comet.{ContribServiceFiles, ContribServices} import org.apache.comet.util.ClassLoaders /** @@ -339,34 +335,8 @@ class CometScanContribSuite extends AnyFunSuite { } } - /** - * Writes a `META-INF/services/org.apache.comet.rules.CometScanContrib` naming `impls` into a - * temp directory, loads it through an isolated child `URLClassLoader`, and hands the discovered - * instances to `f`. The child loader keeps the discovery out of the JVM-wide registry that the - * default-build test above asserts is empty. - */ - private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = { - // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; - // ensure it before createTempDirectory (which requires its parent to exist). - val baseTmp = new File(System.getProperty("java.io.tmpdir")) - baseTmp.mkdirs() - val svcDir = Files.createTempDirectory(baseTmp.toPath, "comet-scan-contrib-test").toFile - try { - val servicesDir = new File(svcDir, "META-INF/services") - assert(servicesDir.mkdirs(), s"could not create $servicesDir") - Files.write( - new File(servicesDir, classOf[CometScanContrib].getName).toPath, - implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) - - f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) - } finally { - def del(file: File): Unit = { - Option(file.listFiles()).foreach(_.foreach(del)) - file.delete() - } - del(svcDir) - } - } + private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = + ContribServiceFiles.withServiceFile(classOf[CometScanContrib], implNames)(f) } /** Inert leaf plan standing in for whatever node a real contrib would return. */ diff --git a/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala new file mode 100644 index 00000000000..a78fd4fa05c --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import java.util.ServiceLoader + +import scala.jdk.CollectionConverters._ + +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, Expression} + +import org.apache.comet.ContribServiceFiles + +/** + * Unit coverage for [[CometExpressionContrib]]: how contrib serdes merge with core's, how they + * are discovered, and how [[QueryPlanSerde.serdeFor]] consults them. + */ +class CometExpressionContribSuite extends AnyFunSuite { + + private val core: Set[Class[_]] = Set(classOf[Add]) + + private def contrib( + serdes: (Class[_ <: Expression], CometExpressionSerde[_])*): CometExpressionContrib = + new CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + serdes.toMap + } + + test("empty when no contribs") { + assert(CometExpressionContrib.merge(core, Nil).isEmpty) + + // A default build ships no service file, so the production registry is empty too. Raw + // discovery is checked first because the registry also comes out empty when a listed + // provider fails to load. + val discovered = + ServiceLoader.load(classOf[CometExpressionContrib], getClass.getClassLoader).asScala.toSeq + assert(discovered.isEmpty, s"unexpected providers: ${discovered.map(_.getClass.getName)}") + assert(CometExpressionContrib.registered.isEmpty) + } + + test("contrib serde is used for an unclaimed class") { + val serde = new StubSerde + val merged = + CometExpressionContrib.merge(core, Seq(contrib(classOf[ContribOnlyExpression] -> serde))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq serde) + } + + test("core wins over a contrib") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq(contrib(classOf[Add] -> new StubSerde, classOf[ContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[Add])) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_ eq kept)) + } + + test("two contribs claiming one class both lose") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + contrib(classOf[ContribOnlyExpression] -> new StubSerde), + contrib( + classOf[ContribOnlyExpression] -> new StubSerde, + classOf[OtherContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[ContribOnlyExpression])) + assert(merged.keySet == Set(classOf[OtherContribOnlyExpression])) + assert(merged(classOf[OtherContribOnlyExpression]) eq kept) + } + + test("a throwing contrib is skipped and others kept") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + new ThrowingExpressionContrib, + new VersionSkewedExpressionContrib, + contrib(classOf[ContribOnlyExpression] -> kept))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq kept) + } + + test("discovery via loadFrom") { + val impls = Seq(classOf[DiscoverableExpressionContrib].getName) + ContribServiceFiles.withServiceFile(classOf[CometExpressionContrib], impls) { loader => + val discovered = CometExpressionContrib.loadContribs(loader) + assert(discovered.map(_.getClass) == Seq(classOf[DiscoverableExpressionContrib])) + val merged = CometExpressionContrib.merge(core, discovered) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_.isInstanceOf[StubSerde])) + } + } + + test("serdeFor falls back to the contrib registry") { + val contribSerde = new StubSerde + val coreSerde = QueryPlanSerde.exprSerdeMap(classOf[Add]) + val contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> contribSerde, classOf[Add] -> new StubSerde) + + assert( + QueryPlanSerde + .serdeFor(classOf[ContribOnlyExpression], contribSerdes) + .exists(_ eq contribSerde)) + assert(QueryPlanSerde.serdeFor(classOf[Add], contribSerdes).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[OtherContribOnlyExpression], contribSerdes).isEmpty) + // The production lookup on a default build sees core serdes only. + assert(QueryPlanSerde.serdeFor(classOf[Add]).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[ContribOnlyExpression]).isEmpty) + } +} + +/** Expression classes core never registers; only their `Class` objects are used. */ +abstract class ContribOnlyExpression extends Expression +abstract class OtherContribOnlyExpression extends Expression + +/** Serde stand-in; tests compare instances by identity. */ +class StubSerde extends CometExpressionSerde[Expression] { + override def convert( + expr: Expression, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = None +} + +/** Public no-arg contrib for ServiceLoader discovery. */ +class DiscoverableExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> new StubSerde) +} + +class ThrowingExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new IllegalStateException("contrib failed to build its serde map") +} + +/** A contrib jar built against Comet internals that have since moved. */ +class VersionSkewedExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new NoSuchMethodError("org.apache.comet.serde.Moved.method()V") +} From dcbbdec8a880fb0973e18153a752ccdf97fb4a66 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Wed, 11 Nov 2026 23:20:41 +0700 Subject: [PATCH 5/8] feat: run Delta's InterleaveBits natively Registers a serde for Delta's InterleaveBits through the contrib expression hook, so the Z-order key for one or two integer inputs runs in Comet's native kernel instead of falling back. Other forms keep falling back with a reason, and spark.comet.expression.InterleaveBits.enabled turns it off. The new suite checks exact bytes for column and literal inputs, nulls, signed extrema and generated pairs against Spark under both of Delta's interleave code paths, across several batches and on an empty table. It replaces the standalone reference checks under dev/repros/delta. The Delta user guide gains a section on Z-order keys, and the scan flag's description now says only scans are left to Spark by default. --- contrib/delta-spark/README.md | 8 +- ....apache.comet.serde.CometExpressionContrib | 17 ++ .../delta/CometDeltaInterleaveBits.scala | 62 +++++ .../delta/DeltaExpressionContrib.scala | 35 +++ .../comet/contrib/delta/DeltaScanConf.scala | 2 +- .../contrib/delta/DeltaScanContrib.scala | 2 +- .../delta/CometDeltaInterleaveBitsSuite.scala | 252 ++++++++++++++++++ .../delta/DeltaInterleaveBitsChecks.scala | 148 ---------- dev/repros/delta/README-7573.md | 70 ----- docs/source/user-guide/latest/delta.md | 21 +- 10 files changed, 392 insertions(+), 225 deletions(-) create mode 100644 contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib create mode 100644 contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala create mode 100644 contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala create mode 100644 contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala delete mode 100644 dev/repros/delta/DeltaInterleaveBitsChecks.scala delete mode 100644 dev/repros/delta/README-7573.md diff --git a/contrib/delta-spark/README.md b/contrib/delta-spark/README.md index 89664d3d61a..c5eae13491f 100644 --- a/contrib/delta-spark/README.md +++ b/contrib/delta-spark/README.md @@ -23,13 +23,17 @@ Native Delta Lake reads for Comet. Delta tables are scanned through Comet's existing native Parquet reader, so they get row-group pruning, page-index pruning, and filter pushdown, with deletion vectors applied inside the scan. -Support is experimental and explicitly opt-in. Two things are required: +The jar also gives Comet a Rust implementation of Delta's `InterleaveBits` +(the Z-order key) for one or two integer inputs. It is on whenever the jar is +present; `spark.comet.expression.InterleaveBits.enabled=false` turns it off. + +Scan support is experimental and explicitly opt-in. Two things are required: 1. This module's jar (`comet-contrib-delta-spark`) on the classpath, alongside `delta-spark`. It is never bundled into `comet-spark`; without it, Comet has no Delta surface at all. 2. `spark.comet.scan.delta.enabled=true`. The default is `false`, so the jar - alone does nothing. + alone leaves Delta table scans to Spark. Unsupported tables and features fall back to Spark's reader. See the [user guide](https://datafusion.apache.org/comet/user-guide/latest/delta.html) diff --git a/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib b/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib new file mode 100644 index 00000000000..18ceec03627 --- /dev/null +++ b/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +org.apache.comet.contrib.delta.DeltaExpressionContrib diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala new file mode 100644 index 00000000000..dadae979309 --- /dev/null +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.types.{BinaryType, IntegerType} + +import org.apache.comet.serde.{CometExpressionSerde, Compatible, SupportLevel, Unsupported} +import org.apache.comet.serde.ExprOuterClass.Expr +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, scalarFunctionExprToProtoWithReturnType} + +/** + * Delta's `InterleaveBits`, the Z-order key `OPTIMIZE ... ZORDER BY` sorts on, through the native + * `delta_interleave_bits` kernel. The kernel takes one or two int inputs; other shapes fall back. + * + * A null input counts as 0 and the result is never null, so the call is not wrapped in a null + * short circuit. + */ +object CometDeltaInterleaveBits extends CometExpressionSerde[InterleaveBits] { + + private val MaxInputs = 2 + + override def getSupportLevel(expr: InterleaveBits): SupportLevel = { + val arity = expr.children.length + if (arity < 1 || arity > MaxInputs) { + Unsupported(Some(s"InterleaveBits supports 1 or $MaxInputs inputs, got $arity")) + } else { + expr.children.find(_.dataType != IntegerType) match { + case Some(child) => + Unsupported( + Some(s"InterleaveBits supports only int inputs, got ${child.dataType.catalogString}")) + case None => Compatible(None) + } + } + } + + override def convert( + expr: InterleaveBits, + inputs: Seq[Attribute], + binding: Boolean): Option[Expr] = { + val args = expr.children.map(exprToProtoInternal(_, inputs, binding)) + scalarFunctionExprToProtoWithReturnType("delta_interleave_bits", BinaryType, false, args: _*) + } +} diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala new file mode 100644 index 00000000000..e94938f3027 --- /dev/null +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.delta.expressions.InterleaveBits + +import org.apache.comet.serde.{CometExpressionContrib, CometExpressionSerde} + +/** + * Serdes for Delta expressions, discovered by core's ServiceLoader (see + * `META-INF/services/org.apache.comet.serde.CometExpressionContrib`). + */ +class DeltaExpressionContrib extends CometExpressionContrib { + + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[InterleaveBits] -> CometDeltaInterleaveBits) +} diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala index c94e01493dc..1986a5d0c20 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala @@ -46,7 +46,7 @@ object DeltaScanConf { "planned by delta-spark are executed through Comet's native Parquet scan, " + "inheriting row-group pruning, page-index pruning, and filter pushdown, with " + "deletion vectors applied inside the scan. Experimental: defaults to false, so " + - "adding the contrib jar does not by itself change how any query is read.") + "adding the contrib jar does not by itself change how any Delta table is scanned.") .booleanConf .createWithDefault(false) diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala index f34bcaec110..fedcac8c81d 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala @@ -62,7 +62,7 @@ class DeltaScanContrib extends CometScanContrib with Logging { if (!DeltaScanConf.scanEnabled) { // Deliberate deviation from the "own but cannot handle => claim" contract: a - // user-disabled contrib must be fully inert (the jar alone changes nothing) and must + // user-disabled contrib must be fully inert (the jar alone changes no scan) and must // not shadow another registered Delta contrib. Tag the opt-in hint for EXPLAIN, pass. withFallbackReason( scanExec, diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala new file mode 100644 index 00000000000..8863f1fca33 --- /dev/null +++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import scala.util.Random + +import org.apache.spark.sql.{Column, DataFrame, Row} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.{BinaryType, IntegerType, StructField, StructType} + +import org.apache.comet.CometConf +import org.apache.comet.serde.Unsupported + +/** + * Delta's `InterleaveBits` through the contrib serde and the native kernel. Delta picks a fast or + * a default implementation when the expression is built, so every query is built and checked + * under both settings. + */ +class CometDeltaInterleaveBitsSuite extends CometDeltaTestBase { + + private type Pair = (Integer, Integer) + + private val x = UnresolvedAttribute("x") + private val y = UnresolvedAttribute("y") + + private def interleave(children: Expression*): Column = + getColumnFromExpression(InterleaveBits(children)) + + private def hex(bytes: Array[Byte]): String = bytes.map(b => f"${b & 0xff}%02X").mkString + + /** Runs `f` with Delta's fast interleave path on, then off. */ + private def withBothDeltaPaths(f: Boolean => Unit): Unit = + Seq(true, false).foreach { fast => + withSQLConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key -> fast.toString)(f(fast)) + } + + /** Asserts every `InterleaveBits` in `df` was built with Delta's fast path set to `fast`. */ + private def assertDeltaPath(df: DataFrame, fast: Boolean): Unit = { + // Delta reads the flag when the expression is built, so the field shows which path ran. + val field = classOf[InterleaveBits].getDeclaredField("fastInterleaveBitsEnabled") + field.setAccessible(true) + val nodes = df.queryExecution.optimizedPlan.flatMap(_.expressions.flatMap(_.collect { + case e: InterleaveBits => e + })) + assert(nodes.nonEmpty, s"no InterleaveBits in\n${df.queryExecution.optimizedPlan}") + nodes.foreach(e => assert(field.getBoolean(e) == fast)) + } + + /** A Parquet table with int columns `x` and `y` holding `pairs`. */ + private def withPairs(pairs: Seq[Pair])(f: DataFrame => Unit): Unit = + withTempPath { dir => + val schema = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + val rows = pairs.map { case (a, b) => Row(a, b) } + spark + .createDataFrame(spark.sparkContext.parallelize(rows, 2), schema) + .write + .parquet(dir.getCanonicalPath) + f(spark.read.parquet(dir.getCanonicalPath)) + } + + private val extrema: Seq[Integer] = Seq(Int.MinValue, Int.MaxValue, 0, -1, null) + + /** Every pair of extrema and nulls, plus seeded random pairs with some nulls. */ + private val generatedPairs: Seq[Pair] = { + val random = new Random(7573L) + def value(i: Int, nullEvery: Int): Integer = + if (i % nullEvery == 0) null else Integer.valueOf(random.nextInt()) + val corners = for (a <- extrema; b <- extrema) yield (a, b) + corners ++ (0 until 1000).map(i => (value(i, 17), value(i, 19))) + } + + test("oracle bytes") { + val pairs: Seq[Pair] = Seq((0x000ff0ff, 0xfff00f00), (-1, null), (null, -1)) + val expected = Set( + ("555555AAAA55AAAA", "000FF0FF", "FFF00F00"), + ("AAAAAAAAAAAAAAAA", "FFFFFFFF", "00000000"), + ("5555555555555555", "00000000", "FFFFFFFF")) + withPairs(pairs) { df => + withBothDeltaPaths { fast => + val query = + df.select(interleave(x, y).as("xy"), interleave(x).as("x1"), interleave(y).as("y1")) + assertDeltaPath(query, fast) + assert(query.schema("xy").dataType == BinaryType && !query.schema("xy").nullable) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + val actual = query.collect().map { r => + ( + hex(r.getAs[Array[Byte]](0)), + hex(r.getAs[Array[Byte]](1)), + hex(r.getAs[Array[Byte]](2))) + } + assert(actual.length == pairs.length) + assert(actual.toSet == expected) + } + } + } + + /** A small batch size, so each task's rows span several Arrow batches. */ + private val smallBatches = CometConf.COMET_BATCH_SIZE.key -> "100" + + test("nulls, extrema, generated pairs") { + withPairs(generatedPairs) { df => + withBothDeltaPaths { fast => + withSQLConf(smallBatches) { + val query = df.select( + col("x"), + col("y"), + interleave(x, y).as("xy"), + interleave(y, x).as("yx"), + interleave(x).as("x1")) + assertDeltaPath(query, fast) + checkSparkAnswerAndOperator(query) + } + } + } + } + + test("column and literal mix") { + withPairs(generatedPairs) { df => + withBothDeltaPaths { fast => + withSQLConf(smallBatches) { + val query = df.select( + col("x"), + col("y"), + interleave(x, Literal(-1)).as("x_lit"), + interleave(Literal(-1), y).as("lit_y")) + assertDeltaPath(query, fast) + assert(!query.schema("x_lit").nullable && !query.schema("lit_y").nullable) + checkSparkAnswerAndOperator(query) + } + } + } + } + + test("empty table") { + withPairs(Seq.empty) { df => + withBothDeltaPaths { fast => + val query = df.select(interleave(x, y).as("xy"), interleave(x).as("x1")) + assertDeltaPath(query, fast) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + assert(query.collect().isEmpty) + } + } + } + + test("all literal inputs") { + val excludeConstantFolding = + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" + // Not foldable, so excluding ConstantFolding must not change the plan or the answer. + Seq(Seq.empty, Seq(excludeConstantFolding)).foreach { extraConf => + withPairs(generatedPairs.take(10)) { df => + withBothDeltaPaths { fast => + withSQLConf(extraConf: _*) { + val query = df.select( + col("x"), + interleave(Literal(1), Literal(2)).as("one_two"), + interleave(Literal(null, IntegerType), Literal(-1)).as("null_minus_one"), + interleave(Literal(0x000ff0ff)).as("one_input")) + assertDeltaPath(query, fast) + assert(!query.schema("null_minus_one").nullable) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + val actual = query.collect() + assert(actual.length == 10) + actual.foreach { r => + assert(hex(r.getAs[Array[Byte]](1)) == "0000000000000006") + assert(hex(r.getAs[Array[Byte]](2)) == "5555555555555555") + assert(hex(r.getAs[Array[Byte]](3)) == "000FF0FF") + } + } + } + } + } + } + + test("falls back for 3 inputs") { + withPairs(generatedPairs.take(50)) { df => + withBothDeltaPaths { _ => + checkSparkAnswerAndFallbackReason( + df.select(interleave(x, y, x).as("xyx")), + "InterleaveBits supports 1 or 2 inputs, got 3") + } + } + } + + test("support level rejects a non-int child") { + assert( + CometDeltaInterleaveBits + .getSupportLevel(InterleaveBits(Seq(Literal(1L)))) + .isInstanceOf[Unsupported]) + assert( + CometDeltaInterleaveBits + .getSupportLevel(InterleaveBits(Seq.empty)) + .isInstanceOf[Unsupported]) + } + + test("disabled by config") { + val key = CometConf.getExprEnabledConfigKey("InterleaveBits") + withPairs(generatedPairs.take(50)) { df => + withSQLConf(key -> "false") { + checkSparkAnswerAndFallbackReason( + df.select(interleave(x, y).as("xy")), + s"Expression support is disabled. Set $key=true") + } + } + } + + test("OPTIMIZE ZORDER BY still matches Spark (Spark JVM path)") { + // The Z-order inputs here are Delta's range-partition expressions, which Comet does not + // convert, so this projection falls back to Spark. It checks that OPTIMIZE still works. + withTempPath { dir => + val path = dir.getCanonicalPath + spark + .range(0, 200, 1, 4) + .selectExpr("cast(id % 13 as int) as a", "cast(id % 7 as int) as b", "id") + .write + .format("delta") + .save(path) + val before = spark.read.format("delta").load(path).collect().toSeq + sql(s"OPTIMIZE delta.`$path` ZORDER BY (a, b)") + val lastOperation = + sql(s"DESCRIBE HISTORY delta.`$path`").orderBy(col("version").desc).head() + assert(lastOperation.getAs[String]("operation") == "OPTIMIZE") + val zOrderBy = lastOperation + .getAs[scala.collection.Map[String, String]]("operationParameters") + .get("zOrderBy") + assert(zOrderBy.exists(v => v.contains("a") && v.contains("b")), s"zOrderBy: $zOrderBy") + checkAnswer(spark.read.format("delta").load(path), before) + } + } +} diff --git a/dev/repros/delta/DeltaInterleaveBitsChecks.scala b/dev/repros/delta/DeltaInterleaveBitsChecks.scala deleted file mode 100644 index eb830223b8b..00000000000 --- a/dev/repros/delta/DeltaInterleaveBitsChecks.scala +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.comet.repro - - -import java.nio.charset.StandardCharsets.UTF_8 -import java.nio.file.{Files, Paths} -import org.apache.spark.sql.{Column, DataFrame, Row, SparkSession} -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Literal} -import org.apache.spark.sql.classic.ClassicConversions._ - -import org.apache.spark.sql.delta.expressions.InterleaveBits -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types._ - -/** Selected reference checks for #7573; no expression implementation is added. */ -object DeltaInterleaveBitsChecks { - def main(args: Array[String]): Unit = { - require(args.length == 2, "usage: stock|comet ") - val mode = args(0); require(Set("stock", "comet")(mode)) - val out = Paths.get(args(1)); Files.createDirectory(out) - val spark = SparkSession.builder - .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension" + - (if (mode == "comet") ",org.apache.comet.CometSparkSessionExtensions" else "")) - .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog") - .config("spark.comet.enabled", (mode == "comet").toString) - .config("spark.comet.exec.enabled", (mode == "comet").toString) - .config("spark.comet.explain.fallback.enabled", "true") - .config("spark.shuffle.manager", if (mode == "comet") - "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" - else "org.apache.spark.shuffle.sort.SortShuffleManager") - .config("spark.memory.offHeap.enabled", "true").config("spark.memory.offHeap.size", "512m") - .config("spark.sql.adaptive.enabled", "false").config("spark.sql.shuffle.partitions", "2") - .getOrCreate() - import spark.implicits._ - var passed = 0 - def check(name: String)(body: => Unit): Unit = { - try { body; Files.write(out.resolve(name + ".passed"), Array.emptyByteArray); passed += 1 } - catch { case e: Throwable => Files.write(out.resolve(name + ".error.txt"), e.toString.getBytes(UTF_8)); throw e } - } - def query(name: String, df: DataFrame)(assertRows: Array[Row] => Unit): Unit = check(name) { - val rows = df.collect(); assertRows(rows) - Files.write(out.resolve(name + ".rows.json"), rows.map(_.json).sorted.mkString("[", ",", "]").getBytes(UTF_8)) - Files.write(out.resolve(name + ".schema.json"), df.schema.json.getBytes(UTF_8)) - Files.write(out.resolve(name + ".plan.txt"), df.queryExecution.executedPlan.toString.getBytes(UTF_8)) - } - try { - val control = spark.range(20).selectExpr("sum(id) total") - query("control", control) { rows => require(rows.length == 1 && rows.head.getLong(0) == 190L) - if (mode == "comet") require(control.queryExecution.executedPlan.toString.contains("Comet")) } - def oracle(values: Seq[java.lang.Integer]): Array[Byte] = { - val bytes = new Array[Byte](4 * values.size); var at = 0 - for (bit <- 31 to 0 by -1; v <- values) { val n = if (v == null) 0 else v.intValue() - if (((n >>> bit) & 1) != 0) bytes(at / 8) = (bytes(at / 8) | (1 << (7-at%8))).toByte - at += 1 } - bytes - } - def hexBytes(b: Array[Byte]): String = b.map(x => f"${x & 255}%02X").mkString - val x = 0x000ff0ff; val y = 0xfff00f00 - check("interleave_asymmetric_oracle") { - require(hexBytes(oracle(Seq(x,y))) == "555555AAAA55AAAA") - require(!oracle(Seq(x,y)).sameElements(oracle(Seq(y,x)))) - require(hexBytes(oracle(Seq(-1,null))) == "AAAAAAAAAAAAAAAA") - require(hexBytes(oracle(Seq(null,-1))) == "5555555555555555") - require(!InterleaveBits(Seq(Literal("x"))).checkInputDataTypes().isSuccess) - } - val random = new scala.util.Random(7573L) - val pairs = Seq[(java.lang.Integer,java.lang.Integer)]((x,y),(y,x),(null,-1),(-1,null),(Int.MinValue,Int.MaxValue),(0,0)) ++ - (0 until 256).map(i => (if (i % 17 == 0) null else Integer.valueOf(random.nextInt()), if (i % 19 == 0) null else Integer.valueOf(random.nextInt()))) - val pairSchema = StructType(Seq(StructField("x",IntegerType),StructField("y",IntegerType))) - val pairFrame = spark.createDataFrame(spark.sparkContext.parallelize(pairs.map {case (a,b) => Row(a,b)},2),pairSchema) - val pairPath = out.resolve("pairs").toString; pairFrame.write.parquet(pairPath) - val inputs = spark.read.parquet(pairPath) - val expectedPairCounts = pairs.groupBy(p => p).map { case (pair, values) => pair -> values.size } - def assertInputPairs(rows: Array[Row]): Unit = { - val actual = rows.toSeq.map(r => ( - if (r.isNullAt(0)) null else Integer.valueOf(r.getInt(0)), - if (r.isNullAt(1)) null else Integer.valueOf(r.getInt(1)))) - require(actual.groupBy(p => p).map { case (pair, values) => pair -> values.size } == expectedPairCounts, - "input pairs changed, duplicated or disappeared") - } - def checkMixedRows(rows: Array[Row], literalFirst: Boolean): Unit = { - require(rows.length == pairs.size, "mixed expression output cardinality changed") - assertInputPairs(rows) - rows.foreach { r => - val value = if (literalFirst) {if (r.isNullAt(1)) null else Integer.valueOf(r.getInt(1))} else {if (r.isNullAt(0)) null else Integer.valueOf(r.getInt(0))} - require(r.getAs[Array[Byte]](2).sameElements(oracle(if (literalFirst) Seq(Integer.valueOf(-1),value) else Seq(value,Integer.valueOf(-1))))) } - } - check("interleave_mixed_validator_rejects_incomplete") { - for (bad <- Seq(Array.empty[Row],Array(Row(x,y,oracle(Seq(Integer.valueOf(x),Integer.valueOf(-1))))), Array.fill(pairs.size)(Row(x,y,oracle(Seq(Integer.valueOf(x),Integer.valueOf(-1))))))) { - var rejected = false - try checkMixedRows(bad,false) catch { case _: IllegalArgumentException => rejected = true } - assert(rejected,"mixed validator accepted empty, truncated or duplicated output") - } - } - for (fast <- Seq(false,true)) { - spark.conf.set(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key,fast.toString) - for (dims <- Seq(1,2)) { - val key = Column(InterleaveBits(Seq("x","y").take(dims).map(UnresolvedAttribute(_)))).as("key") - val df = inputs.select(col("x"),col("y"),key) - require(df.schema("key").dataType == BinaryType && !df.schema("key").nullable) - query(s"interleave_columns_${dims}_$fast",df)(rows => {require(rows.length == pairs.size);assertInputPairs(rows);rows.foreach { r => - val values = (0 until dims).map(i => if (r.isNullAt(i)) null else Integer.valueOf(r.getInt(i))) - require(r.getAs[Array[Byte]](2).sameElements(oracle(values))) }}) - } - for (literalFirst <- Seq(false,true)) { - val children = if (literalFirst) Seq(Literal(-1),UnresolvedAttribute("y")) else Seq(UnresolvedAttribute("x"),Literal(-1)) - val df = inputs.select(col("x"),col("y"),Column(InterleaveBits(children)).as("key")) - require(df.schema("key").dataType == BinaryType && !df.schema("key").nullable) - query(s"interleave_mixed_${literalFirst}_$fast",df)(rows => checkMixedRows(rows,literalFirst)) - } - spark.conf.set("spark.sql.optimizer.excludedRules","org.apache.spark.sql.catalyst.optimizer.ConstantFolding") - for (dims <- Seq(1,2)) { - val literal = inputs.limit(1).select(Column(InterleaveBits(Seq(x,y).take(dims).map(Literal(_)))).as("key")) - require(literal.queryExecution.optimizedPlan.exists(p => p.expressions.exists(_.exists(_.isInstanceOf[InterleaveBits]))),"literal expression folded despite exclusion") - require(literal.schema("key").dataType == BinaryType && !literal.schema("key").nullable) - query(s"interleave_literals_unfolded_${dims}_$fast",literal)(rows => { - require(rows.length == 1) - require(rows.head.getAs[Array[Byte]](0).sameElements(oracle(Seq(x,y).take(dims).map(Integer.valueOf)))) - }) - } - spark.conf.unset("spark.sql.optimizer.excludedRules") - } - require(passed == 15, s"expected 15 named cases, observed $passed") - Files.write(out.resolve("summary.json"),s"{\"mode\":\"$mode\",\"passed\":$passed,\"native_acceptance\":false}".getBytes(UTF_8)) - println(s"PASS DeltaInterleaveBitsChecks $mode cases=$passed (reference/fallback only)") - } finally spark.stop() - } -} diff --git a/dev/repros/delta/README-7573.md b/dev/repros/delta/README-7573.md deleted file mode 100644 index 7d0ae70ab07..00000000000 --- a/dev/repros/delta/README-7573.md +++ /dev/null @@ -1,70 +0,0 @@ - - -# Reference checks for #7573 - -Run commands from `dev/repros/delta`. - -Standalone, test-only probes related to [apache/datafusion-comet#7573](https://github.com/apache/datafusion-comet/issues/7573). They do not implement the requested Comet expression or change admission rules. The issue remains open. - -Run with Spark 4.2.0, Scala 2.13.18 and Java 17. Supply matching Delta Spark and storage JARs, version 4.4.0 or 4.4.1. Use a fresh output directory for every invocation. - -The setup requires `curl`. Set `SPARK_HOME` and `JAVA_HOME` to your local installations before running it. The Comet artifact is `org.apache.datafusion:comet-spark-spark4.2_2.13:1.2.0`; match the Spark 4.2 and Scala 2.13 lines. This assembly contains the platform library used by the validated Linux x86_64 run; no separate Comet companion JAR was used. Other platforms remain unverified. - -```bash -set -eu -: "${SPARK_HOME:?Set SPARK_HOME to a Spark 4.2.0 installation}" -: "${JAVA_HOME:?Set JAVA_HOME to a Java 17 installation}" -export PATH="$JAVA_HOME/bin:$PATH" -DELTA_VERSION=4.4.0 # Use 4.4.1 to repeat the other validated pairing. -RUN_DIR=$(mktemp -d /tmp/comet-interleave.XXXXXX) -ARTIFACT_DIR="$RUN_DIR/artifacts" -mkdir "$ARTIFACT_DIR" -export DELTA_SPARK_JAR="$ARTIFACT_DIR/delta-spark_2.13-$DELTA_VERSION.jar" -export DELTA_STORAGE_JAR="$ARTIFACT_DIR/delta-storage-$DELTA_VERSION.jar" -export COMET_JAR="$ARTIFACT_DIR/comet-spark-spark4.2_2.13-1.2.0.jar" -curl --fail --location --retry 3 -o "$DELTA_SPARK_JAR" \ - "https://repo.maven.apache.org/maven2/io/delta/delta-spark_2.13/$DELTA_VERSION/delta-spark_2.13-$DELTA_VERSION.jar" -curl --fail --location --retry 3 -o "$DELTA_STORAGE_JAR" \ - "https://repo.maven.apache.org/maven2/io/delta/delta-storage/$DELTA_VERSION/delta-storage-$DELTA_VERSION.jar" -curl --fail --location --retry 3 -o "$COMET_JAR" \ - https://repo.maven.apache.org/maven2/org/apache/datafusion/comet-spark-spark4.2_2.13/1.2.0/comet-spark-spark4.2_2.13-1.2.0.jar -mkdir "$RUN_DIR/classes" -java -Xmx1g -cp "$SPARK_HOME/jars/*" scala.tools.nsc.Main \ - -classpath "$SPARK_HOME/jars/*:$DELTA_SPARK_JAR:$DELTA_STORAGE_JAR" \ - -d "$RUN_DIR/classes" DeltaInterleaveBitsChecks.scala -jar cf "$RUN_DIR/checks.jar" -C "$RUN_DIR/classes" . -"$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ - --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR" \ - --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ - "$RUN_DIR/checks.jar" stock "$RUN_DIR/stock" -"$SPARK_HOME/bin/spark-submit" --master 'local[2]' --driver-memory 2g \ - --jars "$DELTA_SPARK_JAR,$DELTA_STORAGE_JAR,$COMET_JAR" \ - --class org.apache.spark.sql.comet.repro.DeltaInterleaveBitsChecks \ - "$RUN_DIR/checks.jar" comet "$RUN_DIR/comet" -``` - -Expected: 15 named cases, matching row/schema JSON across stock and Comet runs, and `native_acceptance: false` in `summary.json`. The Comet lane requires an accelerated control query; passing Delta probes can still execute in Spark. Execution plans are saved beside rows. These are manually run probes, not tests wired into the Comet CI reactor. - -Grounding: selected equivalents of the following versioned upstream tests, not a replacement for their full suites: - -- [InterleaveBitsSuite (v4.4.0)](https://github.com/delta-io/delta/blob/v4.4.0/spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsSuite.scala) -- [InterleaveBitsSuite (v4.4.1)](https://github.com/delta-io/delta/blob/v4.4.1/spark/src/test/scala/org/apache/spark/sql/delta/expressions/InterleaveBitsSuite.scala) - -Validation uses Comet 1.2.0, not a build of this branch. Current-main expression support and the production image remain unverified. diff --git a/docs/source/user-guide/latest/delta.md b/docs/source/user-guide/latest/delta.md index 2d8d7872b30..b25d38d5ef2 100644 --- a/docs/source/user-guide/latest/delta.md +++ b/docs/source/user-guide/latest/delta.md @@ -24,14 +24,14 @@ delta-spark run through Comet's native Parquet scan, inheriting row-group pruning, page-index pruning, and filter pushdown, with deletion vectors applied inside the scan. -Support is experimental and explicitly opt-in. Two things are required: +Scan support is experimental and explicitly opt-in. Two things are required: 1. The `comet-contrib-delta-spark` contrib jar on the classpath, alongside `delta-spark`. It is never bundled into `comet-spark`, and no release publishes it yet, so build it from source with the steps in the [contrib module README](https://github.com/apache/datafusion-comet/blob/main/contrib/delta-spark/README.md#building-and-testing). 2. `spark.comet.scan.delta.enabled=true`. The default is `false`, so - the jar alone does nothing. + the jar alone leaves Delta table scans to Spark. Unsupported tables and features fall back to Spark's reader. See the [contrib module README](https://github.com/apache/datafusion-comet/blob/main/contrib/delta-spark/README.md) @@ -55,6 +55,21 @@ onward, and timestamps from 1900-01-01T00:00:00Z onward (Spark's own rebase cutoff). Disable `spark.comet.scan.delta.enabled` for such tables to read them through Spark. +## Z-Order Keys + +With the contrib jar on the classpath, Comet also runs Delta's `InterleaveBits` +expression, the Z-order key that `OPTIMIZE ... ZORDER BY` builds, in the Comet +pipeline. It is Rust-implemented for one or two integer inputs and returns the +same bytes as Delta, whichever implementation +`spark.databricks.delta.optimize.zorder.fastInterleaveBits.enabled` selects. +Calls with more inputs fall back to Spark. This does not depend on +`spark.comet.scan.delta.enabled`. To turn it off, set +`spark.comet.expression.InterleaveBits.enabled=false`. + +`OPTIMIZE ... ZORDER BY` does not benefit yet. There the inputs of +`InterleaveBits` are Delta's range-partition expressions, which Comet does not +support, so that projection still falls back to Spark. + ## Configuration When Spark splits a Parquet file into several tasks, each native Delta scan builds From 27895228a9598749715c50f065bde1c56c514c36 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Wed, 11 Nov 2026 23:24:23 +0700 Subject: [PATCH 6/8] test: check that delta_interleave_bits ignores the value under a null Arrow builders leave zero under a null slot, so the existing null tests passed even without the kernel's null check. The new test puts -1 under a null and expects the bytes of a zero input. --- .../src/delta_funcs/interleave_bits.rs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/native/spark-expr/src/delta_funcs/interleave_bits.rs b/native/spark-expr/src/delta_funcs/interleave_bits.rs index 3480a13a249..172f0133f6d 100644 --- a/native/spark-expr/src/delta_funcs/interleave_bits.rs +++ b/native/spark-expr/src/delta_funcs/interleave_bits.rs @@ -147,6 +147,7 @@ fn interleave(values: &[u32]) -> u64 { mod tests { use super::*; use arrow::array::ArrayRef; + use arrow::buffer::{NullBuffer, ScalarBuffer}; use arrow::datatypes::Field; use datafusion::config::ConfigOptions; @@ -337,6 +338,35 @@ mod tests { assert_eq!(rows, ["AAAAAAAAAAAAAAAA", "5555555555555555"]); } + #[test] + fn null_slot_ignores_its_value() { + // The value buffer holds -1 under the null slot, unlike the 0 a builder writes there. + let under_null = || { + ColumnarValue::Array(Arc::new(Int32Array::new( + ScalarBuffer::from(vec![-1, -1]), + Some(NullBuffer::from(vec![true, false])), + ))) + }; + let minus_one = || int_array(vec![Some(-1), Some(-1)]); + let cases = [ + ( + vec![minus_one(), under_null()], + ["FFFFFFFFFFFFFFFF", "AAAAAAAAAAAAAAAA"], + ), + ( + vec![under_null(), minus_one()], + ["FFFFFFFFFFFFFFFF", "5555555555555555"], + ), + (vec![under_null()], ["FFFFFFFF", "00000000"]), + ]; + for (args, expected) in cases { + let ColumnarValue::Array(result) = invoke(args, 2).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(to_hex_rows(&result), expected); + } + } + #[test] fn one_input_is_big_endian() { let rows = hex_rows(vec![vec![Some(i32::MIN), Some(-1), Some(0x01020304)]]); From 0a38dcb091209d8fb1f12e4423a079ac585f9002 Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Thu, 12 Nov 2026 17:21:43 -0300 Subject: [PATCH 7/8] test(delta): cover empty interleave batches --- .../src/delta_funcs/interleave_bits.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/native/spark-expr/src/delta_funcs/interleave_bits.rs b/native/spark-expr/src/delta_funcs/interleave_bits.rs index 172f0133f6d..c63cf418d0d 100644 --- a/native/spark-expr/src/delta_funcs/interleave_bits.rs +++ b/native/spark-expr/src/delta_funcs/interleave_bits.rs @@ -373,6 +373,25 @@ mod tests { assert_eq!(rows, ["80000000", "FFFFFFFF", "01020304"]); } + #[test] + fn empty_batches() { + let empty = || int_array(vec![]); + let scalar = || ColumnarValue::Scalar(ScalarValue::Int32(Some(-1))); + for args in [ + vec![empty()], + vec![empty(), empty()], + vec![empty(), scalar()], + vec![scalar(), empty()], + ] { + let ColumnarValue::Array(result) = invoke(args, 0).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(result.data_type(), &DataType::Binary); + assert_eq!(result.len(), 0); + assert_eq!(result.null_count(), 0); + } + } + #[test] fn scalar_broadcasts() { let column = vec![Some(1), Some(2), None]; From c641b0fd834d3a002c6982009425cd5de842799a Mon Sep 17 00:00:00 2001 From: Erik Bogado Date: Thu, 12 Nov 2026 17:21:43 -0300 Subject: [PATCH 8/8] test(delta): finish interleave validation --- contrib/delta-spark/README.md | 20 +++++++ .../delta/CometDeltaInterleaveBitsSuite.scala | 2 +- .../CometDeltaInterleaveBitsBenchmark.scala | 53 +++++++++++++++++++ docs/source/user-guide/latest/delta.md | 6 +-- 4 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala diff --git a/contrib/delta-spark/README.md b/contrib/delta-spark/README.md index c5eae13491f..37a6d92e851 100644 --- a/contrib/delta-spark/README.md +++ b/contrib/delta-spark/README.md @@ -72,3 +72,23 @@ no Docker daemon is reachable; setting `COMET_DELTA_S3_REQUIRED=1`, as the `dev/` contains a benchmark script (`bench_delta_comet.py`) and a harness for running Delta's own test suites against Comet (`run-delta-regression.sh`). + +### InterleaveBits benchmark + +Run from the repository root with JDK 17 and the Rust toolchain available: + +```shell +(cd native && cargo build --release) +./mvnw -Pspark-4.1,delta -Prelease install -DskipTests +COMET_BENCHMARK_JVM_ARGS=$(./mvnw -q -Pspark-4.1 help:evaluate -DforceStdout -Dexpression=extraJavaTestArgs) +COMET_CONF_DIR="$PWD/conf" \ + ./mvnw -f contrib/delta-spark/pom.xml -Pspark-4.1,delta -Prelease \ + exec:exec -Dexec.executable=java -Dexec.classpathScope=test \ + "-Dexec.args=-Xmx6g $COMET_BENCHMARK_JVM_ARGS -Djava.library.path=$PWD/native/target/release -classpath %classpath org.apache.spark.sql.benchmark.CometDeltaInterleaveBitsBenchmark" +``` + +This compares one and two integer inputs over one million Parquet rows, including +nulls in the second input, with Delta's fast implementation enabled and disabled. +Both engines consume the projected keys. Check the output for fallback warnings +before interpreting timings. Results include scanning and projection; they do +not measure `OPTIMIZE` or the kernel in isolation. diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala index 8863f1fca33..fff47cd8e84 100644 --- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala +++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala @@ -245,7 +245,7 @@ class CometDeltaInterleaveBitsSuite extends CometDeltaTestBase { val zOrderBy = lastOperation .getAs[scala.collection.Map[String, String]]("operationParameters") .get("zOrderBy") - assert(zOrderBy.exists(v => v.contains("a") && v.contains("b")), s"zOrderBy: $zOrderBy") + assert(zOrderBy.contains("""["a","b"]"""), s"zOrderBy: $zOrderBy") checkAnswer(spark.read.format("delta").load(path), before) } } diff --git a/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala b/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala new file mode 100644 index 00000000000..84442a6270e --- /dev/null +++ b/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.benchmark + +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** Compares Delta's two JVM implementations with Comet over the same Parquet input. */ +object CometDeltaInterleaveBitsBenchmark extends CometBenchmarkBase { + override def runCometBenchmark(mainArgs: Array[String]): Unit = { + spark.sessionState.functionRegistry.createOrReplaceTempFunction( + "delta_interleave_bits", + children => InterleaveBits(children), + "internal") + runBenchmarkWithTable("Delta InterleaveBits", 1024 * 1024) { rows => + withTempPath { dir => + withTempTable("parquetV1Table") { + prepareTable( + dir, + spark.sql(s"SELECT cast(value as int) AS x, " + + s"IF(value % 5 = 0, cast(NULL as int), cast(shiftright(value, 32) as int)) AS y FROM $tbl")) + Seq(true, false).foreach { fast => + withSQLConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key -> fast.toString) { + Seq("x", "x, y").foreach { inputs => + runExpressionBenchmark( + s"InterleaveBits($inputs), Delta fast=$fast", + rows.toLong, + s"SELECT delta_interleave_bits($inputs) FROM parquetV1Table") + } + } + } + } + } + } + } +} diff --git a/docs/source/user-guide/latest/delta.md b/docs/source/user-guide/latest/delta.md index b25d38d5ef2..08fbfd20f4a 100644 --- a/docs/source/user-guide/latest/delta.md +++ b/docs/source/user-guide/latest/delta.md @@ -66,9 +66,9 @@ Calls with more inputs fall back to Spark. This does not depend on `spark.comet.scan.delta.enabled`. To turn it off, set `spark.comet.expression.InterleaveBits.enabled=false`. -`OPTIMIZE ... ZORDER BY` does not benefit yet. There the inputs of -`InterleaveBits` are Delta's range-partition expressions, which Comet does not -support, so that projection still falls back to Spark. +`OPTIMIZE ... ZORDER BY` uses Delta's range-partition expressions as inputs to +`InterleaveBits`. Comet does not support those inputs, so the key projection +falls back to Spark. ## Configuration