From 3927f04161234ab6c760b5c8831bfcbcd4c8d1e9 Mon Sep 17 00:00:00 2001 From: Dustin Smith Date: Wed, 11 Nov 2026 22:19:29 +0700 Subject: [PATCH] feat: let contribs register expression serdes Adds CometExpressionContrib, a ServiceLoader hook beside CometScanContrib, so a contrib can register a serde for an expression class core never names. Delta's InterleaveBits is the first user: the class lives in delta-spark, which core does not depend on. QueryPlanSerde.serdeFor now looks in core's map first and the contrib registry second, and both lookup sites use it. A contrib entry for a class core already handles is ignored, a class claimed by two contribs is dropped from both since ServiceLoader order is unspecified, and a contrib that throws is skipped. Default builds ship no service file, so nothing changes for them. The scan suite's service-file test helper moves into a shared test utility that both suites use. --- .github/workflows/pr_build_linux.yml | 1 + .github/workflows/pr_build_macos.yml | 1 + .../org/apache/comet/ContribServices.scala | 5 +- .../comet/serde/CometExpressionContrib.scala | 119 +++++++++++++ .../apache/comet/serde/CometScalaUDF.scala | 2 +- .../apache/comet/serde/QueryPlanSerde.scala | 21 ++- .../apache/comet/ContribServiceFiles.scala | 58 +++++++ .../comet/rules/CometScanContribSuite.scala | 36 +--- .../serde/CometExpressionContribSuite.scala | 157 ++++++++++++++++++ 9 files changed, 362 insertions(+), 38 deletions(-) create mode 100644 spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala create mode 100644 spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala create mode 100644 spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index b176bc829ad..1febe7a57f4 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -688,6 +688,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index a9b1a12832d..f539f28d417 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -326,6 +326,7 @@ jobs: org.apache.comet.CometJsonJvmSuite org.apache.comet.SparkErrorConverterSuite org.apache.comet.serde.SerdeRegistrationSuite + org.apache.comet.serde.CometExpressionContribSuite org.apache.comet.expressions.conditional.CometIfSuite org.apache.comet.expressions.conditional.CometCoalesceSuite org.apache.comet.expressions.conditional.CometCaseWhenSuite diff --git a/spark/src/main/scala/org/apache/comet/ContribServices.scala b/spark/src/main/scala/org/apache/comet/ContribServices.scala index 879ed2baac2..0a912d007bf 100644 --- a/spark/src/main/scala/org/apache/comet/ContribServices.scala +++ b/spark/src/main/scala/org/apache/comet/ContribServices.scala @@ -30,7 +30,7 @@ import org.apache.comet.util.ClassLoaders /** * Shared `ServiceLoader` discovery for Comet's optional, out-of-tree contrib registries - * (`CometScanContrib`, `CometConfigProvider`, `PlanDataInjector`). + * (`CometScanContrib`, `CometExpressionContrib`, `CometConfigProvider`, `PlanDataInjector`). * * Exists because the obvious spellings are both wrong in ways that are invisible until a contrib * jar is misbuilt in production: @@ -53,8 +53,7 @@ import org.apache.comet.util.ClassLoaders * changes. * * Not `private[comet]`: `PlanDataInjector` lives in `org.apache.spark.sql.comet`, which is a - * different package root, so a comet-scoped qualifier would hide this from one of its three call - * sites. + * different package root, so a comet-scoped qualifier would hide this from one of its call sites. */ object ContribServices extends Logging { diff --git a/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala new file mode 100644 index 00000000000..26ff4227fb7 --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/serde/CometExpressionContrib.scala @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import scala.util.control.NonFatal + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Expression + +import org.apache.comet.ContribServices + +/** + * Hook that lets an optional, out-of-tree contrib register serdes for expression classes core + * does not know about. Core names no contrib: implementations are discovered at runtime via the + * JDK [[java.util.ServiceLoader]], so default builds (which ship no service file) see an empty + * registry. + * + * A contrib ships an implementation of this trait and a + * `META-INF/services/org.apache.comet.serde.CometExpressionContrib` resource naming it. + * + * Core serdes always win: [[QueryPlanSerde.serdeFor]] consults core first, and a contrib entry + * for a class core registers is ignored. A class that two contribs both claim is dropped from + * both, because ServiceLoader order is not specified and so neither can be preferred. + * + * `expressionSerdes` runs while the registry is being built, so it must not call + * [[QueryPlanSerde.serdeFor]] or otherwise convert expressions. + */ +trait CometExpressionContrib { + + /** Serdes this contrib provides, keyed by the expression class each one converts. */ + def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] +} + +object CometExpressionContrib extends Logging { + + /** + * Contrib serdes for classes core does not register, discovered through the thread context + * ClassLoader (see `ContribServices.load`). Empty on a default build. + */ + private[comet] lazy val registered: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + merge( + QueryPlanSerde.exprSerdeMap.keySet.toSet[Class[_]], + ContribServices.load(classOf[CometExpressionContrib], getClass.getClassLoader)) + + /** Discover implementations visible to `loader`, without touching `registered`. */ + private[comet] def loadContribs(loader: ClassLoader): Seq[CometExpressionContrib] = + ContribServices.loadFrom(classOf[CometExpressionContrib], loader) + + /** + * Combine the serdes of `contribs`, dropping classes in `core` and classes more than one + * contrib claims. A contrib whose `expressionSerdes` throws is skipped with a warning, so one + * broken contrib jar neither fails planning nor hides the serdes of the others. As in + * `CometScanContrib`, a `LinkageError` (a contrib built against other Comet internals) is + * contained the same way, while fatal errors propagate. + */ + private[comet] def merge(core: Set[Class[_]], contribs: Seq[CometExpressionContrib]) + : Map[Class[_ <: Expression], CometExpressionSerde[_]] = { + val claims = contribs.flatMap { contrib => + serdesOf(contrib).map { case (cls, serde) => Claim(cls, contrib.getClass.getName, serde) } + } + val kept = claims.groupBy(_.cls).values.toList.flatMap { entries => + val cls = entries.head.cls + val owners = entries.map(_.owner).mkString(", ") + if (core.contains(cls)) { + logWarning( + s"Ignoring the ${cls.getName} serde from $owners: Comet already has a serde for " + + "this expression.") + None + } else if (entries.size > 1) { + logWarning( + s"Ignoring every serde for ${cls.getName}: more than one contrib registers one " + + s"($owners), and their discovery order is not specified.") + None + } else { + entries.headOption + } + } + kept.map(claim => claim.cls -> claim.serde).toMap + } + + private case class Claim( + cls: Class[_ <: Expression], + owner: String, + serde: CometExpressionSerde[_]) + + private def serdesOf( + contrib: CometExpressionContrib): Seq[(Class[_ <: Expression], CometExpressionSerde[_])] = + try contrib.expressionSerdes.toList + catch { + case NonFatal(e) => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed", + e) + Nil + case e: LinkageError => + logWarning( + s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed with " + + s"${e.getClass.getName}, so it was likely built against a different Comet version", + e) + Nil + } +} diff --git a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala index 7244875e2cb..6596a69eacd 100644 --- a/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala +++ b/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala @@ -94,7 +94,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] { */ private def isEnabled(guard: If): Boolean = { def disabled(e: Expression): Boolean = - QueryPlanSerde.exprSerdeMap.get(e.getClass).exists { serde => + QueryPlanSerde.serdeFor(e.getClass).exists { serde => !CometConf.isExprEnabled( serde.asInstanceOf[CometExpressionSerde[Expression]].getExprConfigName(e)) } diff --git a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala index c8b89b0742c..1f40a861ca1 100644 --- a/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala +++ b/spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala @@ -413,6 +413,25 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { val exprSerdeMap: Map[Class[_ <: Expression], CometExpressionSerde[_]] = serdeGroups.map(_._2).reduce(_ ++ _) + /** + * The serde for expression class `cls`: core's if it has one, otherwise one registered by a + * [[CometExpressionContrib]]. + */ + def serdeFor(cls: Class[_]): Option[CometExpressionSerde[_]] = + serdeFor(cls, CometExpressionContrib.registered) + + /** `serdeFor` against an explicit contrib registry, so tests need not load the real one. */ + private[comet] def serdeFor( + cls: Class[_], + contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]]) + : Option[CometExpressionSerde[_]] = { + // The maps' key type is invariant in its bound; a non-Expression class just misses. + val key = cls.asInstanceOf[Class[_ <: Expression]] + val core = exprSerdeMap.get(key) + // An explicit branch rather than `orElse`, which would allocate a closure on every lookup. + if (core.isDefined) core else contribSerdes.get(key) + } + /** * Mapping of Spark aggregate expression class to Comet expression handler. */ @@ -1120,7 +1139,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim { exprToProtoInternal(child, inputs, binding) case expr => - QueryPlanSerde.exprSerdeMap.get(expr.getClass) match { + QueryPlanSerde.serdeFor(expr.getClass) match { case Some(handler) => convert(expr, handler.asInstanceOf[CometExpressionSerde[Expression]]) case _ => diff --git a/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala new file mode 100644 index 00000000000..40f0c7cf254 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +import java.io.File +import java.net.URLClassLoader +import java.nio.charset.StandardCharsets +import java.nio.file.Files + +/** Test support for driving real `ServiceLoader` discovery of contrib registries. */ +object ContribServiceFiles { + + /** + * Writes a `META-INF/services/` file naming `implNames` into a temp directory and + * hands `f` an isolated child `URLClassLoader` that sees it. The child loader keeps the + * discovery out of the JVM-wide registries, which the default-build tests assert are empty. + */ + def withServiceFile(service: Class[_], implNames: Seq[String])(f: ClassLoader => Unit): Unit = { + // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; + // ensure it before createTempDirectory (which requires its parent to exist). + val baseTmp = new File(System.getProperty("java.io.tmpdir")) + baseTmp.mkdirs() + val svcDir = + Files.createTempDirectory(baseTmp.toPath, s"comet-${service.getSimpleName}-test").toFile + try { + val servicesDir = new File(svcDir, "META-INF/services") + assert(servicesDir.mkdirs(), s"could not create $servicesDir") + Files.write( + new File(servicesDir, service.getName).toPath, + implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) + + f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) + } finally { + def del(file: File): Unit = { + Option(file.listFiles()).foreach(_.foreach(del)) + file.delete() + } + del(svcDir) + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala index 4e41a9b147a..ffbe593591d 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometScanContribSuite.scala @@ -19,10 +19,6 @@ package org.apache.comet.rules -import java.io.File -import java.net.URLClassLoader -import java.nio.charset.StandardCharsets -import java.nio.file.Files import java.util.ServiceLoader import scala.annotation.nowarn @@ -43,7 +39,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, LeafExecNode, SparkPl import org.apache.spark.sql.execution.datasources.HadoopFsRelation import org.apache.spark.sql.execution.datasources.v2.BatchScanExec -import org.apache.comet.ContribServices +import org.apache.comet.{ContribServiceFiles, ContribServices} import org.apache.comet.util.ClassLoaders /** @@ -339,34 +335,8 @@ class CometScanContribSuite extends AnyFunSuite { } } - /** - * Writes a `META-INF/services/org.apache.comet.rules.CometScanContrib` naming `impls` into a - * temp directory, loads it through an isolated child `URLClassLoader`, and hands the discovered - * instances to `f`. The child loader keeps the discovery out of the JVM-wide registry that the - * default-build test above asserts is empty. - */ - private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = { - // The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet; - // ensure it before createTempDirectory (which requires its parent to exist). - val baseTmp = new File(System.getProperty("java.io.tmpdir")) - baseTmp.mkdirs() - val svcDir = Files.createTempDirectory(baseTmp.toPath, "comet-scan-contrib-test").toFile - try { - val servicesDir = new File(svcDir, "META-INF/services") - assert(servicesDir.mkdirs(), s"could not create $servicesDir") - Files.write( - new File(servicesDir, classOf[CometScanContrib].getName).toPath, - implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8)) - - f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader)) - } finally { - def del(file: File): Unit = { - Option(file.listFiles()).foreach(_.foreach(del)) - file.delete() - } - del(svcDir) - } - } + private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = + ContribServiceFiles.withServiceFile(classOf[CometScanContrib], implNames)(f) } /** Inert leaf plan standing in for whatever node a real contrib would return. */ diff --git a/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala new file mode 100644 index 00000000000..a78fd4fa05c --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/serde/CometExpressionContribSuite.scala @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.serde + +import java.util.ServiceLoader + +import scala.jdk.CollectionConverters._ + +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.spark.sql.catalyst.expressions.{Add, Attribute, Expression} + +import org.apache.comet.ContribServiceFiles + +/** + * Unit coverage for [[CometExpressionContrib]]: how contrib serdes merge with core's, how they + * are discovered, and how [[QueryPlanSerde.serdeFor]] consults them. + */ +class CometExpressionContribSuite extends AnyFunSuite { + + private val core: Set[Class[_]] = Set(classOf[Add]) + + private def contrib( + serdes: (Class[_ <: Expression], CometExpressionSerde[_])*): CometExpressionContrib = + new CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + serdes.toMap + } + + test("empty when no contribs") { + assert(CometExpressionContrib.merge(core, Nil).isEmpty) + + // A default build ships no service file, so the production registry is empty too. Raw + // discovery is checked first because the registry also comes out empty when a listed + // provider fails to load. + val discovered = + ServiceLoader.load(classOf[CometExpressionContrib], getClass.getClassLoader).asScala.toSeq + assert(discovered.isEmpty, s"unexpected providers: ${discovered.map(_.getClass.getName)}") + assert(CometExpressionContrib.registered.isEmpty) + } + + test("contrib serde is used for an unclaimed class") { + val serde = new StubSerde + val merged = + CometExpressionContrib.merge(core, Seq(contrib(classOf[ContribOnlyExpression] -> serde))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq serde) + } + + test("core wins over a contrib") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq(contrib(classOf[Add] -> new StubSerde, classOf[ContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[Add])) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_ eq kept)) + } + + test("two contribs claiming one class both lose") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + contrib(classOf[ContribOnlyExpression] -> new StubSerde), + contrib( + classOf[ContribOnlyExpression] -> new StubSerde, + classOf[OtherContribOnlyExpression] -> kept))) + assert(!merged.contains(classOf[ContribOnlyExpression])) + assert(merged.keySet == Set(classOf[OtherContribOnlyExpression])) + assert(merged(classOf[OtherContribOnlyExpression]) eq kept) + } + + test("a throwing contrib is skipped and others kept") { + val kept = new StubSerde + val merged = CometExpressionContrib.merge( + core, + Seq( + new ThrowingExpressionContrib, + new VersionSkewedExpressionContrib, + contrib(classOf[ContribOnlyExpression] -> kept))) + assert(merged.keySet == Set(classOf[ContribOnlyExpression])) + assert(merged(classOf[ContribOnlyExpression]) eq kept) + } + + test("discovery via loadFrom") { + val impls = Seq(classOf[DiscoverableExpressionContrib].getName) + ContribServiceFiles.withServiceFile(classOf[CometExpressionContrib], impls) { loader => + val discovered = CometExpressionContrib.loadContribs(loader) + assert(discovered.map(_.getClass) == Seq(classOf[DiscoverableExpressionContrib])) + val merged = CometExpressionContrib.merge(core, discovered) + assert(merged.get(classOf[ContribOnlyExpression]).exists(_.isInstanceOf[StubSerde])) + } + } + + test("serdeFor falls back to the contrib registry") { + val contribSerde = new StubSerde + val coreSerde = QueryPlanSerde.exprSerdeMap(classOf[Add]) + val contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> contribSerde, classOf[Add] -> new StubSerde) + + assert( + QueryPlanSerde + .serdeFor(classOf[ContribOnlyExpression], contribSerdes) + .exists(_ eq contribSerde)) + assert(QueryPlanSerde.serdeFor(classOf[Add], contribSerdes).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[OtherContribOnlyExpression], contribSerdes).isEmpty) + // The production lookup on a default build sees core serdes only. + assert(QueryPlanSerde.serdeFor(classOf[Add]).exists(_ eq coreSerde)) + assert(QueryPlanSerde.serdeFor(classOf[ContribOnlyExpression]).isEmpty) + } +} + +/** Expression classes core never registers; only their `Class` objects are used. */ +abstract class ContribOnlyExpression extends Expression +abstract class OtherContribOnlyExpression extends Expression + +/** Serde stand-in; tests compare instances by identity. */ +class StubSerde extends CometExpressionSerde[Expression] { + override def convert( + expr: Expression, + inputs: Seq[Attribute], + binding: Boolean): Option[ExprOuterClass.Expr] = None +} + +/** Public no-arg contrib for ServiceLoader discovery. */ +class DiscoverableExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + Map(classOf[ContribOnlyExpression] -> new StubSerde) +} + +class ThrowingExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new IllegalStateException("contrib failed to build its serde map") +} + +/** A contrib jar built against Comet internals that have since moved. */ +class VersionSkewedExpressionContrib extends CometExpressionContrib { + override def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]] = + throw new NoSuchMethodError("org.apache.comet.serde.Moved.method()V") +}