diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index b176bc829ad..1febe7a57f4 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -688,6 +688,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index a9b1a12832d..f539f28d417 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -326,6 +326,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/contrib/delta-spark/README.md b/contrib/delta-spark/README.md index 89664d3d61a..37a6d92e851 100644 --- a/contrib/delta-spark/README.md +++ b/contrib/delta-spark/README.md @@ -23,13 +23,17 @@ Native Delta Lake reads for Comet. Delta tables are scanned through Comet's existing native Parquet reader, so they get row-group pruning, page-index pruning, and filter pushdown, with deletion vectors applied inside the scan. -Support is experimental and explicitly opt-in. Two things are required: +The jar also gives Comet a Rust implementation of Delta's `InterleaveBits` +(the Z-order key) for one or two integer inputs. It is on whenever the jar is +present; `spark.comet.expression.InterleaveBits.enabled=false` turns it off. + +Scan support is experimental and explicitly opt-in. Two things are required: 1. This module's jar (`comet-contrib-delta-spark`) on the classpath, alongside `delta-spark`. It is never bundled into `comet-spark`; without it, Comet has no Delta surface at all. 2. `spark.comet.scan.delta.enabled=true`. The default is `false`, so the jar - alone does nothing. + alone leaves Delta table scans to Spark. Unsupported tables and features fall back to Spark's reader. See the [user guide](https://datafusion.apache.org/comet/user-guide/latest/delta.html) @@ -68,3 +72,23 @@ no Docker daemon is reachable; setting `COMET_DELTA_S3_REQUIRED=1`, as the `dev/` contains a benchmark script (`bench_delta_comet.py`) and a harness for running Delta's own test suites against Comet (`run-delta-regression.sh`). + +### InterleaveBits benchmark + +Run from the repository root with JDK 17 and the Rust toolchain available: + +```shell +(cd native && cargo build --release) +./mvnw -Pspark-4.1,delta -Prelease install -DskipTests +COMET_BENCHMARK_JVM_ARGS=$(./mvnw -q -Pspark-4.1 help:evaluate -DforceStdout -Dexpression=extraJavaTestArgs) +COMET_CONF_DIR="$PWD/conf" \ + ./mvnw -f contrib/delta-spark/pom.xml -Pspark-4.1,delta -Prelease \ + exec:exec -Dexec.executable=java -Dexec.classpathScope=test \ + "-Dexec.args=-Xmx6g $COMET_BENCHMARK_JVM_ARGS -Djava.library.path=$PWD/native/target/release -classpath %classpath org.apache.spark.sql.benchmark.CometDeltaInterleaveBitsBenchmark" +``` + +This compares one and two integer inputs over one million Parquet rows, including +nulls in the second input, with Delta's fast implementation enabled and disabled. +Both engines consume the projected keys. Check the output for fallback warnings +before interpreting timings. Results include scanning and projection; they do +not measure `OPTIMIZE` or the kernel in isolation. diff --git a/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib b/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib new file mode 100644 index 00000000000..18ceec03627 --- /dev/null +++ b/contrib/delta-spark/src/main/resources/META-INF/services/org.apache.comet.serde.CometExpressionContrib @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +org.apache.comet.contrib.delta.DeltaExpressionContrib diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala new file mode 100644 index 00000000000..dadae979309 --- /dev/null +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBits.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.types.{BinaryType, IntegerType} + +import org.apache.comet.serde.{CometExpressionSerde, Compatible, SupportLevel, Unsupported} +import org.apache.comet.serde.ExprOuterClass.Expr +import org.apache.comet.serde.QueryPlanSerde.{exprToProtoInternal, scalarFunctionExprToProtoWithReturnType} + +/** + * Delta's `InterleaveBits`, the Z-order key `OPTIMIZE ... ZORDER BY` sorts on, through the native + * `delta_interleave_bits` kernel. The kernel takes one or two int inputs; other shapes fall back. + * + * A null input counts as 0 and the result is never null, so the call is not wrapped in a null + * short circuit. + */ +object CometDeltaInterleaveBits extends CometExpressionSerde[InterleaveBits] { + + private val MaxInputs = 2 + + override def getSupportLevel(expr: InterleaveBits): SupportLevel = { + val arity = expr.children.length + if (arity < 1 || arity > MaxInputs) { + Unsupported(Some(s"InterleaveBits supports 1 or $MaxInputs inputs, got $arity")) + } else { + expr.children.find(_.dataType != IntegerType) match { + case Some(child) => + Unsupported( + Some(s"InterleaveBits supports only int inputs, got ${child.dataType.catalogString}")) + case None => Compatible(None) + } + } + } + + override def convert( + expr: InterleaveBits, + inputs: Seq[Attribute], + binding: Boolean): Option[Expr] = { + val args = expr.children.map(exprToProtoInternal(_, inputs, binding)) + scalarFunctionExprToProtoWithReturnType("delta_interleave_bits", BinaryType, false, args: _*) + } +} diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala new file mode 100644 index 00000000000..e94938f3027 --- /dev/null +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaExpressionContrib.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.delta.expressions.InterleaveBits + +import org.apache.comet.serde.{CometExpressionContrib, CometExpressionSerde} + +/** + * Serdes for Delta expressions, discovered by core's ServiceLoader (see + * `META-INF/services/org.apache.comet.serde.CometExpressionContrib`). + */ +class DeltaExpressionContrib extends CometExpressionContrib { + + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[InterleaveBits] -> CometDeltaInterleaveBits) +} diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala index c94e01493dc..1986a5d0c20 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanConf.scala @@ -46,7 +46,7 @@ object DeltaScanConf { "planned by delta-spark are executed through Comet's native Parquet scan, " + "inheriting row-group pruning, page-index pruning, and filter pushdown, with " + "deletion vectors applied inside the scan. Experimental: defaults to false, so " + - "adding the contrib jar does not by itself change how any query is read.") + "adding the contrib jar does not by itself change how any Delta table is scanned.") .booleanConf .createWithDefault(false) diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala index f34bcaec110..fedcac8c81d 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanContrib.scala @@ -62,7 +62,7 @@ class DeltaScanContrib extends CometScanContrib with Logging { if (!DeltaScanConf.scanEnabled) { // Deliberate deviation from the "own but cannot handle => claim" contract: a - // user-disabled contrib must be fully inert (the jar alone changes nothing) and must + // user-disabled contrib must be fully inert (the jar alone changes no scan) and must // not shadow another registered Delta contrib. Tag the opt-in hint for EXPLAIN, pass. withFallbackReason( scanExec, diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala new file mode 100644 index 00000000000..fff47cd8e84 --- /dev/null +++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaInterleaveBitsSuite.scala @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import scala.util.Random + +import org.apache.spark.sql.{Column, DataFrame, Row} +import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute +import org.apache.spark.sql.catalyst.expressions.{Expression, Literal} +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.delta.sources.DeltaSQLConf +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.types.{BinaryType, IntegerType, StructField, StructType} + +import org.apache.comet.CometConf +import org.apache.comet.serde.Unsupported + +/** + * Delta's `InterleaveBits` through the contrib serde and the native kernel. Delta picks a fast or + * a default implementation when the expression is built, so every query is built and checked + * under both settings. + */ +class CometDeltaInterleaveBitsSuite extends CometDeltaTestBase { + + private type Pair = (Integer, Integer) + + private val x = UnresolvedAttribute("x") + private val y = UnresolvedAttribute("y") + + private def interleave(children: Expression*): Column = + getColumnFromExpression(InterleaveBits(children)) + + private def hex(bytes: Array[Byte]): String = bytes.map(b => f"${b & 0xff}%02X").mkString + + /** Runs `f` with Delta's fast interleave path on, then off. */ + private def withBothDeltaPaths(f: Boolean => Unit): Unit = + Seq(true, false).foreach { fast => + withSQLConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key -> fast.toString)(f(fast)) + } + + /** Asserts every `InterleaveBits` in `df` was built with Delta's fast path set to `fast`. */ + private def assertDeltaPath(df: DataFrame, fast: Boolean): Unit = { + // Delta reads the flag when the expression is built, so the field shows which path ran. + val field = classOf[InterleaveBits].getDeclaredField("fastInterleaveBitsEnabled") + field.setAccessible(true) + val nodes = df.queryExecution.optimizedPlan.flatMap(_.expressions.flatMap(_.collect { + case e: InterleaveBits => e + })) + assert(nodes.nonEmpty, s"no InterleaveBits in\n${df.queryExecution.optimizedPlan}") + nodes.foreach(e => assert(field.getBoolean(e) == fast)) + } + + /** A Parquet table with int columns `x` and `y` holding `pairs`. */ + private def withPairs(pairs: Seq[Pair])(f: DataFrame => Unit): Unit = + withTempPath { dir => + val schema = + StructType(Seq(StructField("x", IntegerType), StructField("y", IntegerType))) + val rows = pairs.map { case (a, b) => Row(a, b) } + spark + .createDataFrame(spark.sparkContext.parallelize(rows, 2), schema) + .write + .parquet(dir.getCanonicalPath) + f(spark.read.parquet(dir.getCanonicalPath)) + } + + private val extrema: Seq[Integer] = Seq(Int.MinValue, Int.MaxValue, 0, -1, null) + + /** Every pair of extrema and nulls, plus seeded random pairs with some nulls. */ + private val generatedPairs: Seq[Pair] = { + val random = new Random(7573L) + def value(i: Int, nullEvery: Int): Integer = + if (i % nullEvery == 0) null else Integer.valueOf(random.nextInt()) + val corners = for (a <- extrema; b <- extrema) yield (a, b) + corners ++ (0 until 1000).map(i => (value(i, 17), value(i, 19))) + } + + test("oracle bytes") { + val pairs: Seq[Pair] = Seq((0x000ff0ff, 0xfff00f00), (-1, null), (null, -1)) + val expected = Set( + ("555555AAAA55AAAA", "000FF0FF", "FFF00F00"), + ("AAAAAAAAAAAAAAAA", "FFFFFFFF", "00000000"), + ("5555555555555555", "00000000", "FFFFFFFF")) + withPairs(pairs) { df => + withBothDeltaPaths { fast => + val query = + df.select(interleave(x, y).as("xy"), interleave(x).as("x1"), interleave(y).as("y1")) + assertDeltaPath(query, fast) + assert(query.schema("xy").dataType == BinaryType && !query.schema("xy").nullable) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + val actual = query.collect().map { r => + ( + hex(r.getAs[Array[Byte]](0)), + hex(r.getAs[Array[Byte]](1)), + hex(r.getAs[Array[Byte]](2))) + } + assert(actual.length == pairs.length) + assert(actual.toSet == expected) + } + } + } + + /** A small batch size, so each task's rows span several Arrow batches. */ + private val smallBatches = CometConf.COMET_BATCH_SIZE.key -> "100" + + test("nulls, extrema, generated pairs") { + withPairs(generatedPairs) { df => + withBothDeltaPaths { fast => + withSQLConf(smallBatches) { + val query = df.select( + col("x"), + col("y"), + interleave(x, y).as("xy"), + interleave(y, x).as("yx"), + interleave(x).as("x1")) + assertDeltaPath(query, fast) + checkSparkAnswerAndOperator(query) + } + } + } + } + + test("column and literal mix") { + withPairs(generatedPairs) { df => + withBothDeltaPaths { fast => + withSQLConf(smallBatches) { + val query = df.select( + col("x"), + col("y"), + interleave(x, Literal(-1)).as("x_lit"), + interleave(Literal(-1), y).as("lit_y")) + assertDeltaPath(query, fast) + assert(!query.schema("x_lit").nullable && !query.schema("lit_y").nullable) + checkSparkAnswerAndOperator(query) + } + } + } + } + + test("empty table") { + withPairs(Seq.empty) { df => + withBothDeltaPaths { fast => + val query = df.select(interleave(x, y).as("xy"), interleave(x).as("x1")) + assertDeltaPath(query, fast) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + assert(query.collect().isEmpty) + } + } + } + + test("all literal inputs") { + val excludeConstantFolding = + "spark.sql.optimizer.excludedRules" -> + "org.apache.spark.sql.catalyst.optimizer.ConstantFolding" + // Not foldable, so excluding ConstantFolding must not change the plan or the answer. + Seq(Seq.empty, Seq(excludeConstantFolding)).foreach { extraConf => + withPairs(generatedPairs.take(10)) { df => + withBothDeltaPaths { fast => + withSQLConf(extraConf: _*) { + val query = df.select( + col("x"), + interleave(Literal(1), Literal(2)).as("one_two"), + interleave(Literal(null, IntegerType), Literal(-1)).as("null_minus_one"), + interleave(Literal(0x000ff0ff)).as("one_input")) + assertDeltaPath(query, fast) + assert(!query.schema("null_minus_one").nullable) + checkSparkAnswerAndImpl(query, native = Seq("interleavebits")) + val actual = query.collect() + assert(actual.length == 10) + actual.foreach { r => + assert(hex(r.getAs[Array[Byte]](1)) == "0000000000000006") + assert(hex(r.getAs[Array[Byte]](2)) == "5555555555555555") + assert(hex(r.getAs[Array[Byte]](3)) == "000FF0FF") + } + } + } + } + } + } + + test("falls back for 3 inputs") { + withPairs(generatedPairs.take(50)) { df => + withBothDeltaPaths { _ => + checkSparkAnswerAndFallbackReason( + df.select(interleave(x, y, x).as("xyx")), + "InterleaveBits supports 1 or 2 inputs, got 3") + } + } + } + + test("support level rejects a non-int child") { + assert( + CometDeltaInterleaveBits + .getSupportLevel(InterleaveBits(Seq(Literal(1L)))) + .isInstanceOf[Unsupported]) + assert( + CometDeltaInterleaveBits + .getSupportLevel(InterleaveBits(Seq.empty)) + .isInstanceOf[Unsupported]) + } + + test("disabled by config") { + val key = CometConf.getExprEnabledConfigKey("InterleaveBits") + withPairs(generatedPairs.take(50)) { df => + withSQLConf(key -> "false") { + checkSparkAnswerAndFallbackReason( + df.select(interleave(x, y).as("xy")), + s"Expression support is disabled. Set $key=true") + } + } + } + + test("OPTIMIZE ZORDER BY still matches Spark (Spark JVM path)") { + // The Z-order inputs here are Delta's range-partition expressions, which Comet does not + // convert, so this projection falls back to Spark. It checks that OPTIMIZE still works. + withTempPath { dir => + val path = dir.getCanonicalPath + spark + .range(0, 200, 1, 4) + .selectExpr("cast(id % 13 as int) as a", "cast(id % 7 as int) as b", "id") + .write + .format("delta") + .save(path) + val before = spark.read.format("delta").load(path).collect().toSeq + sql(s"OPTIMIZE delta.`$path` ZORDER BY (a, b)") + val lastOperation = + sql(s"DESCRIBE HISTORY delta.`$path`").orderBy(col("version").desc).head() + assert(lastOperation.getAs[String]("operation") == "OPTIMIZE") + val zOrderBy = lastOperation + .getAs[scala.collection.Map[String, String]]("operationParameters") + .get("zOrderBy") + assert(zOrderBy.contains("""["a","b"]"""), s"zOrderBy: $zOrderBy") + checkAnswer(spark.read.format("delta").load(path), before) + } + } +} diff --git a/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala b/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala new file mode 100644 index 00000000000..84442a6270e --- /dev/null +++ b/contrib/delta-spark/src/test/scala/org/apache/spark/sql/benchmark/CometDeltaInterleaveBitsBenchmark.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.benchmark + +import org.apache.spark.sql.delta.expressions.InterleaveBits +import org.apache.spark.sql.delta.sources.DeltaSQLConf + +/** Compares Delta's two JVM implementations with Comet over the same Parquet input. */ +object CometDeltaInterleaveBitsBenchmark extends CometBenchmarkBase { + override def runCometBenchmark(mainArgs: Array[String]): Unit = { + spark.sessionState.functionRegistry.createOrReplaceTempFunction( + "delta_interleave_bits", + children => InterleaveBits(children), + "internal") + runBenchmarkWithTable("Delta InterleaveBits", 1024 * 1024) { rows => + withTempPath { dir => + withTempTable("parquetV1Table") { + prepareTable( + dir, + spark.sql(s"SELECT cast(value as int) AS x, " + + s"IF(value % 5 = 0, cast(NULL as int), cast(shiftright(value, 32) as int)) AS y FROM $tbl")) + Seq(true, false).foreach { fast => + withSQLConf(DeltaSQLConf.FAST_INTERLEAVE_BITS_ENABLED.key -> fast.toString) { + Seq("x", "x, y").foreach { inputs => + runExpressionBenchmark( + s"InterleaveBits($inputs), Delta fast=$fast", + rows.toLong, + s"SELECT delta_interleave_bits($inputs) FROM parquetV1Table") + } + } + } + } + } + } + } +} diff --git a/docs/source/user-guide/latest/delta.md b/docs/source/user-guide/latest/delta.md index 2d8d7872b30..08fbfd20f4a 100644 --- a/docs/source/user-guide/latest/delta.md +++ b/docs/source/user-guide/latest/delta.md @@ -24,14 +24,14 @@ delta-spark run through Comet's native Parquet scan, inheriting row-group pruning, page-index pruning, and filter pushdown, with deletion vectors applied inside the scan. -Support is experimental and explicitly opt-in. Two things are required: +Scan support is experimental and explicitly opt-in. Two things are required: 1. The `comet-contrib-delta-spark` contrib jar on the classpath, alongside `delta-spark`. It is never bundled into `comet-spark`, and no release publishes it yet, so build it from source with the steps in the [contrib module README](https://github.com/apache/datafusion-comet/blob/main/contrib/delta-spark/README.md#building-and-testing). 2. `spark.comet.scan.delta.enabled=true`. The default is `false`, so - the jar alone does nothing. + the jar alone leaves Delta table scans to Spark. Unsupported tables and features fall back to Spark's reader. See the [contrib module README](https://github.com/apache/datafusion-comet/blob/main/contrib/delta-spark/README.md) @@ -55,6 +55,21 @@ onward, and timestamps from 1900-01-01T00:00:00Z onward (Spark's own rebase cutoff). Disable `spark.comet.scan.delta.enabled` for such tables to read them through Spark. +## Z-Order Keys + +With the contrib jar on the classpath, Comet also runs Delta's `InterleaveBits` +expression, the Z-order key that `OPTIMIZE ... ZORDER BY` builds, in the Comet +pipeline. It is Rust-implemented for one or two integer inputs and returns the +same bytes as Delta, whichever implementation +`spark.databricks.delta.optimize.zorder.fastInterleaveBits.enabled` selects. +Calls with more inputs fall back to Spark. This does not depend on +`spark.comet.scan.delta.enabled`. To turn it off, set +`spark.comet.expression.InterleaveBits.enabled=false`. + +`OPTIMIZE ... ZORDER BY` uses Delta's range-partition expressions as inputs to +`InterleaveBits`. Comet does not support those inputs, so the key projection +falls back to Spark. + ## Configuration When Spark splits a Parquet file into several tasks, each native Delta scan builds diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 0086538b276..e7f968166be 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -32,10 +32,10 @@ use crate::{ spark_ceil, spark_day_name, spark_decimal_div, spark_decimal_integral_div, spark_floor, spark_isnan, spark_lpad, spark_make_decimal, spark_month_name, spark_read_side_padding, spark_round, spark_rpad, spark_sequence, spark_sequence_recomputed, spark_to_time, spark_unhex, - spark_unscaled_value, EvalMode, SparkArrayExtrema, SparkArrayPositionFunc, SparkArrayRemove, - SparkArraySetOp, SparkArraySlice, SparkArraySort, SparkArraysOverlap, SparkContains, - SparkDateDiff, SparkDateFromUnixDate, SparkDateTrunc, SparkDayOfWeek, SparkFlatten, - SparkFloatArrayContains, SparkIcebergBucket, SparkIcebergTemporalTransform, + spark_unscaled_value, DeltaInterleaveBits, EvalMode, SparkArrayExtrema, SparkArrayPositionFunc, + SparkArrayRemove, SparkArraySetOp, SparkArraySlice, SparkArraySort, SparkArraysOverlap, + SparkContains, SparkDateDiff, SparkDateFromUnixDate, SparkDateTrunc, SparkDayOfWeek, + SparkFlatten, SparkFloatArrayContains, SparkIcebergBucket, SparkIcebergTemporalTransform, SparkIcebergTruncate, SparkMakeDate, SparkMakeInterval, SparkMakeTime, SparkMapExtract, SparkNextDay, SparkSecondsToTimestamp, SparkSizeFunc, SparkSortArray, SparkWeekDay, }; @@ -365,6 +365,7 @@ fn all_scalar_functions() -> Vec> { Arc::new(ScalarUDF::new_from_impl(SparkDateTrunc::default())), Arc::new(ScalarUDF::new_from_impl(SparkDayOfWeek::default())), Arc::new(ScalarUDF::new_from_impl(SparkWeekDay::default())), + Arc::new(ScalarUDF::new_from_impl(DeltaInterleaveBits::default())), Arc::new(ScalarUDF::new_from_impl(SparkFlatten::default())), Arc::new(ScalarUDF::new_from_impl(SparkIcebergBucket::default())), Arc::new(ScalarUDF::new_from_impl(SparkIcebergTruncate::default())), diff --git a/native/spark-expr/src/delta_funcs/interleave_bits.rs b/native/spark-expr/src/delta_funcs/interleave_bits.rs new file mode 100644 index 00000000000..c63cf418d0d --- /dev/null +++ b/native/spark-expr/src/delta_funcs/interleave_bits.rs @@ -0,0 +1,470 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::array::{Array, AsArray, BinaryArray, BinaryBuilder, Int32Array}; +use arrow::datatypes::{DataType, Int32Type}; +use datafusion::common::{plan_err, Result, ScalarValue}; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, +}; +use std::sync::Arc; + +const NAME: &str = "delta_interleave_bits"; + +/// Largest number of inputs the kernel accepts; two 32-bit inputs fill the `u64` key. +const MAX_INPUTS: usize = 2; + +const BITS_PER_INPUT: u32 = i32::BITS; +const BYTES_PER_INPUT: usize = 4; + +/// Delta Lake's `InterleaveBits`, the Z-order key `OPTIMIZE ... ZORDER BY` sorts on. +/// +/// Takes one or two `Int32` inputs and returns `4 * n` bytes: bits 31 down to 0 of each input's +/// two's-complement value, round-robin in argument order, so the first input supplies the higher +/// bit of each group. A null input counts as 0 and the result is never null. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct DeltaInterleaveBits { + signature: Signature, +} + +impl DeltaInterleaveBits { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Int32]), + TypeSignature::Exact(vec![DataType::Int32, DataType::Int32]), + ], + Volatility::Immutable, + ), + } + } +} + +impl Default for DeltaInterleaveBits { + fn default() -> Self { + Self::new() + } +} + +impl ScalarUDFImpl for DeltaInterleaveBits { + fn name(&self) -> &str { + NAME + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> Result { + check_arg_types(arg_types)?; + Ok(DataType::Binary) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let number_rows = args.number_rows; + let args = args.args; + check_arg_types(&args.iter().map(|a| a.data_type()).collect::>())?; + let all_scalar = args.iter().all(|a| matches!(a, ColumnarValue::Scalar(_))); + let rows = if all_scalar { 1 } else { number_rows }; + let arrays = args + .iter() + .map(|a| a.to_array(rows)) + .collect::>>()?; + let columns = arrays + .iter() + .map(|a| a.as_primitive::()) + .collect::>(); + let result = interleave_columns(&columns, rows); + if all_scalar { + Ok(ColumnarValue::Scalar(ScalarValue::try_from_array( + &result, 0, + )?)) + } else { + Ok(ColumnarValue::Array(Arc::new(result))) + } + } +} + +fn check_arg_types(arg_types: &[DataType]) -> Result<()> { + if !(1..=MAX_INPUTS).contains(&arg_types.len()) { + return plan_err!( + "{NAME} expects 1 to {MAX_INPUTS} arguments, got {}", + arg_types.len() + ); + } + match arg_types.iter().find(|t| **t != DataType::Int32) { + Some(other) => plan_err!("{NAME} expects Int32 input, got {other}"), + None => Ok(()), + } +} + +fn interleave_columns(columns: &[&Int32Array], rows: usize) -> BinaryArray { + let width = BYTES_PER_INPUT * columns.len(); + let mut builder = BinaryBuilder::with_capacity(rows, rows * width); + let mut values = [0u32; MAX_INPUTS]; + for row in 0..rows { + for (value, column) in values.iter_mut().zip(columns) { + *value = if column.is_null(row) { + 0 + } else { + column.value(row) as u32 + }; + } + let key = interleave(&values[..columns.len()]); + builder.append_value(&key.to_be_bytes()[size_of::() - width..]); + } + builder.finish() +} + +/// Interleaves `values` MSB-first, one bit from each value in turn, into the low +/// `32 * values.len()` bits of the key. +fn interleave(values: &[u32]) -> u64 { + let mut key = 0u64; + for pos in (0..BITS_PER_INPUT).rev() { + for &value in values { + key = (key << 1) | u64::from((value >> pos) & 1); + } + } + key +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::ArrayRef; + use arrow::buffer::{NullBuffer, ScalarBuffer}; + use arrow::datatypes::Field; + use datafusion::config::ConfigOptions; + + /// Interleaved keys Delta's own `InterleaveBits.interleaveBits` produced for these inputs, + /// with the fast path both on and off (the two agreed on every row). + const DELTA_GOLDENS: &[(&[i32], &str)] = &[ + (&[i32::MIN], "80000000"), + (&[i32::MAX], "7FFFFFFF"), + (&[0], "00000000"), + (&[-1], "FFFFFFFF"), + (&[1], "00000001"), + (&[1431655765], "55555555"), + (&[-1431655766], "AAAAAAAA"), + (&[1044735], "000FF0FF"), + (&[-1044736], "FFF00F00"), + (&[16909060], "01020304"), + (&[i32::MIN, i32::MIN], "C000000000000000"), + (&[i32::MIN, i32::MAX], "9555555555555555"), + (&[i32::MIN, 0], "8000000000000000"), + (&[i32::MIN, -1], "D555555555555555"), + (&[i32::MIN, 1], "8000000000000001"), + (&[i32::MIN, 1431655765], "9111111111111111"), + (&[i32::MIN, -1431655766], "C444444444444444"), + (&[i32::MIN, 1044735], "8000005555005555"), + (&[i32::MIN, -1044736], "D555550000550000"), + (&[i32::MIN, 16909060], "8001000400050010"), + (&[i32::MAX, i32::MIN], "6AAAAAAAAAAAAAAA"), + (&[i32::MAX, i32::MAX], "3FFFFFFFFFFFFFFF"), + (&[i32::MAX, 0], "2AAAAAAAAAAAAAAA"), + (&[i32::MAX, -1], "7FFFFFFFFFFFFFFF"), + (&[i32::MAX, 1], "2AAAAAAAAAAAAAAB"), + (&[i32::MAX, 1431655765], "3BBBBBBBBBBBBBBB"), + (&[i32::MAX, -1431655766], "6EEEEEEEEEEEEEEE"), + (&[i32::MAX, 1044735], "2AAAAAFFFFAAFFFF"), + (&[i32::MAX, -1044736], "7FFFFFAAAAFFAAAA"), + (&[i32::MAX, 16909060], "2AABAAAEAAAFAABA"), + (&[0, i32::MIN], "4000000000000000"), + (&[0, i32::MAX], "1555555555555555"), + (&[0, 0], "0000000000000000"), + (&[0, -1], "5555555555555555"), + (&[0, 1], "0000000000000001"), + (&[0, 1431655765], "1111111111111111"), + (&[0, -1431655766], "4444444444444444"), + (&[0, 1044735], "0000005555005555"), + (&[0, -1044736], "5555550000550000"), + (&[0, 16909060], "0001000400050010"), + (&[-1, i32::MIN], "EAAAAAAAAAAAAAAA"), + (&[-1, i32::MAX], "BFFFFFFFFFFFFFFF"), + (&[-1, 0], "AAAAAAAAAAAAAAAA"), + (&[-1, -1], "FFFFFFFFFFFFFFFF"), + (&[-1, 1], "AAAAAAAAAAAAAAAB"), + (&[-1, 1431655765], "BBBBBBBBBBBBBBBB"), + (&[-1, -1431655766], "EEEEEEEEEEEEEEEE"), + (&[-1, 1044735], "AAAAAAFFFFAAFFFF"), + (&[-1, -1044736], "FFFFFFAAAAFFAAAA"), + (&[-1, 16909060], "AAABAAAEAAAFAABA"), + (&[1, i32::MIN], "4000000000000002"), + (&[1, i32::MAX], "1555555555555557"), + (&[1, 0], "0000000000000002"), + (&[1, -1], "5555555555555557"), + (&[1, 1], "0000000000000003"), + (&[1, 1431655765], "1111111111111113"), + (&[1, -1431655766], "4444444444444446"), + (&[1, 1044735], "0000005555005557"), + (&[1, -1044736], "5555550000550002"), + (&[1, 16909060], "0001000400050012"), + (&[1431655765, i32::MIN], "6222222222222222"), + (&[1431655765, i32::MAX], "3777777777777777"), + (&[1431655765, 0], "2222222222222222"), + (&[1431655765, -1], "7777777777777777"), + (&[1431655765, 1], "2222222222222223"), + (&[1431655765, 1431655765], "3333333333333333"), + (&[1431655765, -1431655766], "6666666666666666"), + (&[1431655765, 1044735], "2222227777227777"), + (&[1431655765, -1044736], "7777772222772222"), + (&[1431655765, 16909060], "2223222622272232"), + (&[-1431655766, i32::MIN], "C888888888888888"), + (&[-1431655766, i32::MAX], "9DDDDDDDDDDDDDDD"), + (&[-1431655766, 0], "8888888888888888"), + (&[-1431655766, -1], "DDDDDDDDDDDDDDDD"), + (&[-1431655766, 1], "8888888888888889"), + (&[-1431655766, 1431655765], "9999999999999999"), + (&[-1431655766, -1431655766], "CCCCCCCCCCCCCCCC"), + (&[-1431655766, 1044735], "888888DDDD88DDDD"), + (&[-1431655766, -1044736], "DDDDDD8888DD8888"), + (&[-1431655766, 16909060], "8889888C888D8898"), + (&[1044735, i32::MIN], "400000AAAA00AAAA"), + (&[1044735, i32::MAX], "155555FFFF55FFFF"), + (&[1044735, 0], "000000AAAA00AAAA"), + (&[1044735, -1], "555555FFFF55FFFF"), + (&[1044735, 1], "000000AAAA00AAAB"), + (&[1044735, 1431655765], "111111BBBB11BBBB"), + (&[1044735, -1431655766], "444444EEEE44EEEE"), + (&[1044735, 1044735], "000000FFFF00FFFF"), + (&[1044735, -1044736], "555555AAAA55AAAA"), + (&[1044735, 16909060], "000100AEAA05AABA"), + (&[-1044736, i32::MIN], "EAAAAA0000AA0000"), + (&[-1044736, i32::MAX], "BFFFFF5555FF5555"), + (&[-1044736, 0], "AAAAAA0000AA0000"), + (&[-1044736, -1], "FFFFFF5555FF5555"), + (&[-1044736, 1], "AAAAAA0000AA0001"), + (&[-1044736, 1431655765], "BBBBBB1111BB1111"), + (&[-1044736, -1431655766], "EEEEEE4444EE4444"), + (&[-1044736, 1044735], "AAAAAA5555AA5555"), + (&[-1044736, -1044736], "FFFFFF0000FF0000"), + (&[-1044736, 16909060], "AAABAA0400AF0010"), + (&[16909060, i32::MIN], "40020008000A0020"), + (&[16909060, i32::MAX], "1557555D555F5575"), + (&[16909060, 0], "00020008000A0020"), + (&[16909060, -1], "5557555D555F5575"), + (&[16909060, 1], "00020008000A0021"), + (&[16909060, 1431655765], "11131119111B1131"), + (&[16909060, -1431655766], "4446444C444E4464"), + (&[16909060, 1044735], "0002005D550A5575"), + (&[16909060, -1044736], "55575508005F0020"), + (&[16909060, 16909060], "0003000C000F0030"), + (&[-194441980, -713859073], "FB313D8711F75575"), + (&[1465097161, 551116650], "262A734B93DFB4C6"), + (&[-2044496387, -1402509360], "C4781C1F3E37FBA2"), + (&[-42021569, 936478219], "AFB77BA9E0B60AEF"), + (&[-371001683, -1494708188], "EC96FC48EB288CB2"), + (&[-11981971, -1746055248], "EBBF74D318DF6DA2"), + (&[1396948856, -1908031548], "625E301BCFFF7A90"), + (&[-964581762, -717158876], "F1399003883C2EB8"), + (&[-1802157460, 629602743], "8631C23675496DB5"), + (&[47397773, -796979038], "5108B75F0B8AC4A6"), + (&[2119901077, 859130386], "2FAD279B12EE8326"), + (&[-681693820, -2140680892], "E22A36BD58F39030"), + (&[2129441368, -1087931688], "6FFDACB59F4C73C0"), + (&[458445216, -393048690], "56CA630E6253C854"), + (&[1762056806, -117967399], "7DC2553DF7997969"), + (&[1232985102, -1191240484], "65C27FF7A3F051F8"), + (&[-650702380, -866216185], "F2D21B7E4317A235"), + (&[-1061812808, 807767500], "A5008E394073DAD0"), + (&[-293689653, 887702561], "ADB87EE99838A48B"), + (&[1552189062, -1186747436], "67E19025C46DD138"), + ]; + + fn invoke(args: Vec, number_rows: usize) -> Result { + DeltaInterleaveBits::new().invoke_with_args(ScalarFunctionArgs { + args, + arg_fields: vec![], + number_rows, + return_field: Arc::new(Field::new("result", DataType::Binary, false)), + config_options: Arc::new(ConfigOptions::default()), + }) + } + + fn int_array(values: Vec>) -> ColumnarValue { + ColumnarValue::Array(Arc::new(Int32Array::from(values))) + } + + /// Invokes the function on array arguments and returns each row as upper-case hex. + fn hex_rows(columns: Vec>>) -> Vec { + let rows = columns[0].len(); + let args = columns.into_iter().map(int_array).collect(); + let ColumnarValue::Array(result) = invoke(args, rows).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(result.null_count(), 0); + to_hex_rows(&result) + } + + fn to_hex_rows(array: &ArrayRef) -> Vec { + array + .as_binary::() + .iter() + .map(|bytes| to_hex(bytes.expect("result is never null"))) + .collect() + } + + fn to_hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02X}")).collect() + } + + #[test] + fn oracle_two_inputs() { + let rows = hex_rows(vec![ + vec![Some(0x000ff0ff)], + vec![Some(0xfff00f00_u32 as i32)], + ]); + assert_eq!(rows, ["555555AAAA55AAAA"]); + } + + #[test] + fn null_counts_as_zero() { + let rows = hex_rows(vec![vec![Some(-1), None], vec![None, Some(-1)]]); + assert_eq!(rows, ["AAAAAAAAAAAAAAAA", "5555555555555555"]); + } + + #[test] + fn null_slot_ignores_its_value() { + // The value buffer holds -1 under the null slot, unlike the 0 a builder writes there. + let under_null = || { + ColumnarValue::Array(Arc::new(Int32Array::new( + ScalarBuffer::from(vec![-1, -1]), + Some(NullBuffer::from(vec![true, false])), + ))) + }; + let minus_one = || int_array(vec![Some(-1), Some(-1)]); + let cases = [ + ( + vec![minus_one(), under_null()], + ["FFFFFFFFFFFFFFFF", "AAAAAAAAAAAAAAAA"], + ), + ( + vec![under_null(), minus_one()], + ["FFFFFFFFFFFFFFFF", "5555555555555555"], + ), + (vec![under_null()], ["FFFFFFFF", "00000000"]), + ]; + for (args, expected) in cases { + let ColumnarValue::Array(result) = invoke(args, 2).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(to_hex_rows(&result), expected); + } + } + + #[test] + fn one_input_is_big_endian() { + let rows = hex_rows(vec![vec![Some(i32::MIN), Some(-1), Some(0x01020304)]]); + assert_eq!(rows, ["80000000", "FFFFFFFF", "01020304"]); + } + + #[test] + fn empty_batches() { + let empty = || int_array(vec![]); + let scalar = || ColumnarValue::Scalar(ScalarValue::Int32(Some(-1))); + for args in [ + vec![empty()], + vec![empty(), empty()], + vec![empty(), scalar()], + vec![scalar(), empty()], + ] { + let ColumnarValue::Array(result) = invoke(args, 0).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(result.data_type(), &DataType::Binary); + assert_eq!(result.len(), 0); + assert_eq!(result.null_count(), 0); + } + } + + #[test] + fn scalar_broadcasts() { + let column = vec![Some(1), Some(2), None]; + let minus_one = || ColumnarValue::Scalar(ScalarValue::Int32(Some(-1))); + let cases = [ + ( + vec![int_array(column.clone()), minus_one()], + vec![column.clone(), vec![Some(-1); 3]], + ), + ( + vec![minus_one(), int_array(column.clone())], + vec![vec![Some(-1); 3], column.clone()], + ), + ]; + for (args, expanded) in cases { + let ColumnarValue::Array(result) = invoke(args, column.len()).unwrap() else { + panic!("expected an array result"); + }; + assert_eq!(to_hex_rows(&result), hex_rows(expanded)); + } + } + + #[test] + fn all_scalar_args() { + let cases: [Vec>; 5] = [ + vec![Some(0x000ff0ff), Some(0xfff00f00_u32 as i32)], + vec![None, Some(-1)], + vec![None, None], + vec![Some(0x01020304)], + vec![None], + ]; + for values in cases { + let args = values + .iter() + .map(|v| ColumnarValue::Scalar(ScalarValue::Int32(*v))) + .collect(); + let ColumnarValue::Scalar(ScalarValue::Binary(Some(bytes))) = invoke(args, 1).unwrap() + else { + panic!("expected a non-null binary scalar for {values:?}"); + }; + let expected = hex_rows(values.iter().map(|v| vec![*v]).collect()); + assert_eq!(vec![to_hex(&bytes)], expected, "{values:?}"); + } + } + + #[test] + fn delta_goldens() { + for (inputs, expected) in DELTA_GOLDENS { + let columns = inputs.iter().map(|v| vec![Some(*v)]).collect(); + assert_eq!(hex_rows(columns), [*expected], "{inputs:?}"); + } + } + + #[test] + fn rejects_bad_arity() { + let udf = DeltaInterleaveBits::new(); + for arity in [0, 3] { + assert!(udf.return_type(&vec![DataType::Int32; arity]).is_err()); + let args = vec![ColumnarValue::Scalar(ScalarValue::Int32(Some(1))); arity]; + assert!(invoke(args, 1).is_err()); + } + } + + #[test] + fn rejects_non_int32_input() { + let udf = DeltaInterleaveBits::new(); + for arg_types in [ + vec![DataType::Int64], + vec![DataType::Int32, DataType::Int64], + ] { + assert!(udf.return_type(&arg_types).is_err(), "{arg_types:?}"); + } + let args = vec![ColumnarValue::Scalar(ScalarValue::Int64(Some(1)))]; + assert!(invoke(args, 1).is_err()); + } +} diff --git a/native/spark-expr/src/delta_funcs/mod.rs b/native/spark-expr/src/delta_funcs/mod.rs new file mode 100644 index 00000000000..f31b4aad5b5 --- /dev/null +++ b/native/spark-expr/src/delta_funcs/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Native implementations of Delta Lake expressions. + +mod interleave_bits; + +pub use interleave_bits::DeltaInterleaveBits; diff --git a/native/spark-expr/src/lib.rs b/native/spark-expr/src/lib.rs index ea8316bcce5..a557a38c664 100644 --- a/native/spark-expr/src/lib.rs +++ b/native/spark-expr/src/lib.rs @@ -48,8 +48,10 @@ pub mod hash_funcs; mod string_funcs; mod datetime_funcs; +mod delta_funcs; mod iceberg_funcs; pub use agg_funcs::*; +pub use delta_funcs::DeltaInterleaveBits; pub use iceberg_funcs::{SparkIcebergBucket, SparkIcebergTemporalTransform, SparkIcebergTruncate}; pub use cast::{spark_cast, Cast, SparkCastOptions}; diff --git a/spark/src/main/scala/org/apache/comet/ContribServices.scala b/spark/src/main/scala/org/apache/comet/ContribServices.scala index 879ed2baac2..0a912d007bf 100644 --- a/spark/src/main/scala/org/apache/comet/ContribServices.scala +++ b/spark/src/main/scala/org/apache/comet/ContribServices.scala @@ -30,7 +30,7 @@ import org.apache.comet.util.ClassLoaders /** * Shared `ServiceLoader` discovery for Comet's optional, out-of-tree contrib registries - * (`CometScanContrib`, `CometConfigProvider`, `PlanDataInjector`). + * (`CometScanContrib`, `CometExpressionContrib`, `CometConfigProvider`, `PlanDataInjector`). * * Exists because the obvious spellings are both wrong in ways that are invisible until a contrib * jar is misbuilt in production: @@ -53,8 +53,7 @@ import org.apache.comet.util.ClassLoaders * changes. * * Not `private[comet]`: `PlanDataInjector` lives in `org.apache.spark.sql.comet`, which is a - * different package root, so a comet-scoped qualifier would hide this from one of its three call - * sites. + * different package root, so a comet-scoped qualifier would hide this from one of its call sites. */ object ContribServices extends Logging { diff --git a/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala new file mode 100644 index 00000000000..26ff4227fb7 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.util.control.NonFatal + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Expression + +import org.apache.comet.ContribServices + +/** + * Hook that lets an optional, out-of-tree contrib register serdes for expression classes core + * does not know about. Core names no contrib: implementations are discovered at runtime via the + * JDK [[java.util.ServiceLoader]], so default builds (which ship no service file) see an empty + * registry. + * + * A contrib ships an implementation of this trait and a + * `META-INF/services/org.apache.comet.serde.CometExpressionContrib` resource naming it. + * + * Core serdes always win: [[QueryPlanSerde.serdeFor]] consults core first, and a contrib entry + * for a class core registers is ignored. A class that two contribs both claim is dropped from + * both, because ServiceLoader order is not specified and so neither can be preferred. + * + * `expressionSerdes` runs while the registry is being built, so it must not call + * [[QueryPlanSerde.serdeFor]] or otherwise convert expressions. + */ +trait CometExpressionContrib { + + /** Serdes this contrib provides, keyed by the expression class each one converts. */ + def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] +} + +object CometExpressionContrib extends Logging { + + /** + * Contrib serdes for classes core does not register, discovered through the thread context + * ClassLoader (see `ContribServices.load`). Empty on a default build. + */ + private[comet] lazy val registered: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + merge( + QueryPlanSerde.exprSerdeMap.keySet.toSet[Class[_]], + ContribServices.load(classOf[CometExpressionContrib], getClass.getClassLoader)) + + /** Discover implementations visible to `loader`, without touching `registered`. */ + private[comet] def loadContribs(loader: ClassLoader): Seq[CometExpressionContrib] = + ContribServices.loadFrom(classOf[CometExpressionContrib], loader) + + /** + * Combine the serdes of `contribs`, dropping classes in `core` and classes more than one + * contrib claims. A contrib whose `expressionSerdes` throws is skipped with a warning, so one + * broken contrib jar neither fails planning nor hides the serdes of the others. As in + * `CometScanContrib`, a `LinkageError` (a contrib built against other Comet internals) is + * contained the same way, while fatal errors propagate. + */ + private[comet] def merge(core: Set[Class[_]], contribs: Seq[CometExpressionContrib]) + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = { + val claims = contribs.flatMap { contrib => + serdesOf(contrib).map { case (cls, serde) => Claim(cls, contrib.getClass.getName, serde) } + } + val kept = claims.groupBy(_.cls).values.toList.flatMap { entries => + val cls = entries.head.cls + val owners = entries.map(_.owner).mkString(", ") + if (core.contains(cls)) { + logWarning( + s"Ignoring the ${cls.getName} serde from $owners: Comet already has a serde for " + + "this expression.") + None + } else if (entries.size > 1) { + logWarning( + s"Ignoring every serde for ${cls.getName}: more than one contrib registers one " + + s"($owners), and their discovery order is not specified.") + None + } else { + entries.headOption + } + } + kept.map(claim => claim.cls -> claim.serde).toMap + } + + private case class Claim( + cls: Class[_ <: Expression], + owner: String, + serde: CometExpressionSerde[_]) + + private def serdesOf( + contrib: CometExpressionContrib): Seq[(Class[_ <: Expression], CometExpressionSerde[_])] = + try contrib.expressionSerdes.toList + catch { + case NonFatal(e) => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed", + e) + Nil + case e: LinkageError => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed with " + + s"${e.getClass.getName}, so it was likely built against a different Comet version", + e) + Nil + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 7244875e2cb..6596a69eacd 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -94,7 +94,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { */ private def isEnabled(guard: If): Boolean = { def disabled(e: Expression): Boolean = - QueryPlanSerde.exprSerdeMap.get(e.getClass).exists { serde => + QueryPlanSerde.serdeFor(e.getClass).exists { serde => !CometConf.isExprEnabled( serde.asInstanceOf[CometExpressionSerde[Expression]].getExprConfigName(e)) } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8b89b0742c..1f40a861ca1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -413,6 +413,25 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { val exprSerdeMap: Map[Class[_ <: Expression], CometExpressionSerde[_]] = serdeGroups.map(_._2).reduce(_ ++ _) + /** + * The serde for expression class `cls`: core's if it has one, otherwise one registered by a + * [[CometExpressionContrib]]. + */ + def serdeFor(cls: Class[_]): Option[CometExpressionSerde[_]] = + serdeFor(cls, CometExpressionContrib.registered) + + /** `serdeFor` against an explicit contrib registry, so tests need not load the real one. */ + private[comet] def serdeFor( + cls: Class[_], + contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]]) + : Option[CometExpressionSerde[_]] = { + // The maps' key type is invariant in its bound; a non-Expression class just misses. + val key = cls.asInstanceOf[Class[_ <: Expression]] + val core = exprSerdeMap.get(key) + // An explicit branch rather than `orElse`, which would allocate a closure on every lookup. + if (core.isDefined) core else contribSerdes.get(key) + } + /** * Mapping of Spark aggregate expression class to Comet expression handler. */ @@ -1120,7 +1139,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { exprToProtoInternal(child, inputs, binding) case expr => - QueryPlanSerde.exprSerdeMap.get(expr.getClass) match { + QueryPlanSerde.serdeFor(expr.getClass) match { case Some(handler) => convert(expr, handler.asInstanceOf[CometExpressionSerde[Expression]]) case _ => diff --git a/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala new file mode 100644 index 00000000000..40f0c7cf254 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File +import java.net.URLClassLoader +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +/** Test support for driving real `ServiceLoader` discovery of contrib registries. */ +object ContribServiceFiles { + + /** + * Writes a `META-INF/services/` file naming `implNames` into a temp directory and + * hands `f` an isolated child `URLClassLoader` that sees it. The child loader keeps the + * discovery out of the JVM-wide registries, which the default-build tests assert are empty. + */ + def withServiceFile(service: Class[_], implNames: Seq[String])(f: ClassLoader => Unit): Unit = { + // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; + // ensure it before createTempDirectory (which requires its parent to exist). + val baseTmp = new File(System.getProperty("java.io.tmpdir")) + baseTmp.mkdirs() + val svcDir = + Files.createTempDirectory(baseTmp.toPath, s"comet-${service.getSimpleName}-test").toFile + try { + val servicesDir = new File(svcDir, "META-INF/services") + assert(servicesDir.mkdirs(), s"could not create $servicesDir") + Files.write( + new File(servicesDir, service.getName).toPath, + implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) + + f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) + } finally { + def del(file: File): Unit = { + Option(file.listFiles()).foreach(_.foreach(del)) + file.delete() + } + del(svcDir) + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 4e41a9b147a..ffbe593591d 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -19,10 +19,6 @@ package org.apache.comet.rules -import java.io.File -import java.net.URLClassLoader -import java.nio.charset.StandardCharsets -import java.nio.file.Files import java.util.ServiceLoader import scala.annotation.nowarn @@ -43,7 +39,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, LeafExecNode, SparkPl import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.comet.ContribServices +import org.apache.comet.{ContribServiceFiles, ContribServices} import org.apache.comet.util.ClassLoaders /** @@ -339,34 +335,8 @@ class CometScanContribSuite extends AnyFunSuite { } } - /** - * Writes a `META-INF/services/org.apache.comet.rules.CometScanContrib` naming `impls` into a - * temp directory, loads it through an isolated child `URLClassLoader`, and hands the discovered - * instances to `f`. The child loader keeps the discovery out of the JVM-wide registry that the - * default-build test above asserts is empty. - */ - private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = { - // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; - // ensure it before createTempDirectory (which requires its parent to exist). - val baseTmp = new File(System.getProperty("java.io.tmpdir")) - baseTmp.mkdirs() - val svcDir = Files.createTempDirectory(baseTmp.toPath, "comet-scan-contrib-test").toFile - try { - val servicesDir = new File(svcDir, "META-INF/services") - assert(servicesDir.mkdirs(), s"could not create $servicesDir") - Files.write( - new File(servicesDir, classOf[CometScanContrib].getName).toPath, - implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) - - f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) - } finally { - def del(file: File): Unit = { - Option(file.listFiles()).foreach(_.foreach(del)) - file.delete() - } - del(svcDir) - } - } + private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = + ContribServiceFiles.withServiceFile(classOf[CometScanContrib], implNames)(f) } /** Inert leaf plan standing in for whatever node a real contrib would return. */ diff --git a/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala new file mode 100644 index 00000000000..a78fd4fa05c --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import java.util.ServiceLoader + +import scala.jdk.CollectionConverters._ + +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, Expression} + +import org.apache.comet.ContribServiceFiles + +/** + * Unit coverage for [[CometExpressionContrib]]: how contrib serdes merge with core's, how they + * are discovered, and how [[QueryPlanSerde.serdeFor]] consults them. + */ +class CometExpressionContribSuite extends AnyFunSuite { + + private val core: Set[Class[_]] = Set(classOf[Add]) + + private def contrib( + serdes: (Class[_ <: Expression], CometExpressionSerde[_])*): CometExpressionContrib = + new CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + serdes.toMap + } + + test("empty when no contribs") { + assert(CometExpressionContrib.merge(core, Nil).isEmpty) + + // A default build ships no service file, so the production registry is empty too. Raw + // discovery is checked first because the registry also comes out empty when a listed + // provider fails to load. + val discovered = + ServiceLoader.load(classOf[CometExpressionContrib], getClass.getClassLoader).asScala.toSeq + assert(discovered.isEmpty, s"unexpected providers: ${discovered.map(_.getClass.getName)}") + assert(CometExpressionContrib.registered.isEmpty) + } + + test("contrib serde is used for an unclaimed class") { + val serde = new StubSerde + val merged = + CometExpressionContrib.merge(core, Seq(contrib(classOf[ContribOnlyExpression] -> serde))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq serde) + } + + test("core wins over a contrib") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq(contrib(classOf[Add] -> new StubSerde, classOf[ContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[Add])) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_ eq kept)) + } + + test("two contribs claiming one class both lose") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + contrib(classOf[ContribOnlyExpression] -> new StubSerde), + contrib( + classOf[ContribOnlyExpression] -> new StubSerde, + classOf[OtherContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[ContribOnlyExpression])) + assert(merged.keySet == Set(classOf[OtherContribOnlyExpression])) + assert(merged(classOf[OtherContribOnlyExpression]) eq kept) + } + + test("a throwing contrib is skipped and others kept") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + new ThrowingExpressionContrib, + new VersionSkewedExpressionContrib, + contrib(classOf[ContribOnlyExpression] -> kept))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq kept) + } + + test("discovery via loadFrom") { + val impls = Seq(classOf[DiscoverableExpressionContrib].getName) + ContribServiceFiles.withServiceFile(classOf[CometExpressionContrib], impls) { loader => + val discovered = CometExpressionContrib.loadContribs(loader) + assert(discovered.map(_.getClass) == Seq(classOf[DiscoverableExpressionContrib])) + val merged = CometExpressionContrib.merge(core, discovered) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_.isInstanceOf[StubSerde])) + } + } + + test("serdeFor falls back to the contrib registry") { + val contribSerde = new StubSerde + val coreSerde = QueryPlanSerde.exprSerdeMap(classOf[Add]) + val contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> contribSerde, classOf[Add] -> new StubSerde) + + assert( + QueryPlanSerde + .serdeFor(classOf[ContribOnlyExpression], contribSerdes) + .exists(_ eq contribSerde)) + assert(QueryPlanSerde.serdeFor(classOf[Add], contribSerdes).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[OtherContribOnlyExpression], contribSerdes).isEmpty) + // The production lookup on a default build sees core serdes only. + assert(QueryPlanSerde.serdeFor(classOf[Add]).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[ContribOnlyExpression]).isEmpty) + } +} + +/** Expression classes core never registers; only their `Class` objects are used. */ +abstract class ContribOnlyExpression extends Expression +abstract class OtherContribOnlyExpression extends Expression + +/** Serde stand-in; tests compare instances by identity. */ +class StubSerde extends CometExpressionSerde[Expression] { + override def convert( + expr: Expression, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = None +} + +/** Public no-arg contrib for ServiceLoader discovery. */ +class DiscoverableExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> new StubSerde) +} + +class ThrowingExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new IllegalStateException("contrib failed to build its serde map") +} + +/** A contrib jar built against Comet internals that have since moved. */ +class VersionSkewedExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new NoSuchMethodError("org.apache.comet.serde.Moved.method()V") +}