Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/pr_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -700,6 +700,7 @@ jobs:
org.apache.comet.CometJsonJvmSuite
org.apache.comet.SparkErrorConverterSuite
org.apache.comet.serde.SerdeRegistrationSuite
org.apache.comet.serde.CometExpressionContribSuite
org.apache.comet.expressions.conditional.CometIfSuite
org.apache.comet.expressions.conditional.CometCoalesceSuite
org.apache.comet.expressions.conditional.CometCaseWhenSuite
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/pr_build_macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,7 @@ jobs:
org.apache.comet.CometJsonJvmSuite
org.apache.comet.SparkErrorConverterSuite
org.apache.comet.serde.SerdeRegistrationSuite
org.apache.comet.serde.CometExpressionContribSuite
org.apache.comet.expressions.conditional.CometIfSuite
org.apache.comet.expressions.conditional.CometCoalesceSuite
org.apache.comet.expressions.conditional.CometCaseWhenSuite
Expand Down
5 changes: 2 additions & 3 deletions spark/src/main/scala/org/apache/comet/ContribServices.scala
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import org.apache.comet.util.ClassLoaders

/**
* Shared `ServiceLoader` discovery for Comet's optional, out-of-tree contrib registries
* (`CometScanContrib`, `CometConfigProvider`, `PlanDataInjector`).
* (`CometScanContrib`, `CometExpressionContrib`, `CometConfigProvider`, `PlanDataInjector`).
*
* Exists because the obvious spellings are both wrong in ways that are invisible until a contrib
* jar is misbuilt in production:
Expand All @@ -53,8 +53,7 @@ import org.apache.comet.util.ClassLoaders
* changes.
*
* Not `private[comet]`: `PlanDataInjector` lives in `org.apache.spark.sql.comet`, which is a
* different package root, so a comet-scoped qualifier would hide this from one of its three call
* sites.
* different package root, so a comet-scoped qualifier would hide this from one of its call sites.
*/
object ContribServices extends Logging {

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.comet.serde

import scala.util.control.NonFatal

import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.expressions.Expression

import org.apache.comet.ContribServices

/**
* Hook that lets an optional, out-of-tree contrib register serdes for expression classes core
* does not know about. Core names no contrib: implementations are discovered at runtime via the
* JDK [[java.util.ServiceLoader]], so default builds (which ship no service file) see an empty
* registry.
*
* A contrib ships an implementation of this trait and a
* `META-INF/services/org.apache.comet.serde.CometExpressionContrib` resource naming it.
*
* Core serdes always win: [[QueryPlanSerde.serdeFor]] consults core first, and a contrib entry
* for a class core registers is ignored. A class that two contribs both claim is dropped from
* both, because ServiceLoader order is not specified and so neither can be preferred.
*
* `expressionSerdes` runs while the registry is being built, so it must not call
* [[QueryPlanSerde.serdeFor]] or otherwise convert expressions.
*/
trait CometExpressionContrib {

/** Serdes this contrib provides, keyed by the expression class each one converts. */
def expressionSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]]
}

object CometExpressionContrib extends Logging {

/**
* Contrib serdes for classes core does not register, discovered through the thread context
* ClassLoader (see `ContribServices.load`). Empty on a default build.
*/
private[comet] lazy val registered: Map[Class[_ <: Expression], CometExpressionSerde[_]] =
merge(
QueryPlanSerde.exprSerdeMap.keySet.toSet[Class[_]],
ContribServices.load(classOf[CometExpressionContrib], getClass.getClassLoader))

/** Discover implementations visible to `loader`, without touching `registered`. */
private[comet] def loadContribs(loader: ClassLoader): Seq[CometExpressionContrib] =
ContribServices.loadFrom(classOf[CometExpressionContrib], loader)

/**
* Combine the serdes of `contribs`, dropping classes in `core` and classes more than one
* contrib claims. A contrib whose `expressionSerdes` throws is skipped with a warning, so one
* broken contrib jar neither fails planning nor hides the serdes of the others. As in
* `CometScanContrib`, a `LinkageError` (a contrib built against other Comet internals) is
* contained the same way, while fatal errors propagate.
*/
private[comet] def merge(core: Set[Class[_]], contribs: Seq[CometExpressionContrib])
: Map[Class[_ <: Expression], CometExpressionSerde[_]] = {
val claims = contribs.flatMap { contrib =>
serdesOf(contrib).map { case (cls, serde) => Claim(cls, contrib.getClass.getName, serde) }
}
val kept = claims.groupBy(_.cls).values.toList.flatMap { entries =>
val cls = entries.head.cls
val owners = entries.map(_.owner).mkString(", ")
if (core.contains(cls)) {
logWarning(
s"Ignoring the ${cls.getName} serde from $owners: Comet already has a serde for " +
"this expression.")
None
} else if (entries.size > 1) {
logWarning(
s"Ignoring every serde for ${cls.getName}: more than one contrib registers one " +
s"($owners), and their discovery order is not specified.")
None
} else {
entries.headOption
}
}
kept.map(claim => claim.cls -> claim.serde).toMap
}

private case class Claim(
cls: Class[_ <: Expression],
owner: String,
serde: CometExpressionSerde[_])

private def serdesOf(
contrib: CometExpressionContrib): Seq[(Class[_ <: Expression], CometExpressionSerde[_])] =
try contrib.expressionSerdes.toList
catch {
case NonFatal(e) =>
logWarning(
s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed",
e)
Nil
case e: LinkageError =>
logWarning(
s"Skipping contrib ${contrib.getClass.getName}: its expression serdes failed with " +
s"${e.getClass.getName}, so it was likely built against a different Comet version",
e)
Nil
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ object CometScalaUDF extends CometExpressionSerde[ScalaUDF] {
*/
private def isEnabled(guard: If): Boolean = {
def disabled(e: Expression): Boolean =
QueryPlanSerde.exprSerdeMap.get(e.getClass).exists { serde =>
QueryPlanSerde.serdeFor(e.getClass).exists { serde =>
!CometConf.isExprEnabled(
serde.asInstanceOf[CometExpressionSerde[Expression]].getExprConfigName(e))
}
Expand Down
21 changes: 20 additions & 1 deletion spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala
Original file line number Diff line number Diff line change
Expand Up @@ -412,6 +412,25 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
val exprSerdeMap: Map[Class[_ <: Expression], CometExpressionSerde[_]] =
serdeGroups.map(_._2).reduce(_ ++ _)

/**
* The serde for expression class `cls`: core's if it has one, otherwise one registered by a
* [[CometExpressionContrib]].
*/
def serdeFor(cls: Class[_]): Option[CometExpressionSerde[_]] =
serdeFor(cls, CometExpressionContrib.registered)

/** `serdeFor` against an explicit contrib registry, so tests need not load the real one. */
private[comet] def serdeFor(
cls: Class[_],
contribSerdes: Map[Class[_ <: Expression], CometExpressionSerde[_]])
: Option[CometExpressionSerde[_]] = {
// The maps' key type is invariant in its bound; a non-Expression class just misses.
val key = cls.asInstanceOf[Class[_ <: Expression]]
val core = exprSerdeMap.get(key)
// An explicit branch rather than `orElse`, which would allocate a closure on every lookup.
if (core.isDefined) core else contribSerdes.get(key)
}

/**
* Mapping of Spark aggregate expression class to Comet expression handler.
*/
Expand Down Expand Up @@ -1119,7 +1138,7 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
exprToProtoInternal(child, inputs, binding)

case expr =>
QueryPlanSerde.exprSerdeMap.get(expr.getClass) match {
QueryPlanSerde.serdeFor(expr.getClass) match {
case Some(handler) =>
convert(expr, handler.asInstanceOf[CometExpressionSerde[Expression]])
case _ =>
Expand Down
58 changes: 58 additions & 0 deletions spark/src/test/scala/org/apache/comet/ContribServiceFiles.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.comet

import java.io.File
import java.net.URLClassLoader
import java.nio.charset.StandardCharsets
import java.nio.file.Files

/** Test support for driving real `ServiceLoader` discovery of contrib registries. */
object ContribServiceFiles {

/**
* Writes a `META-INF/services/<service>` file naming `implNames` into a temp directory and
* hands `f` an isolated child `URLClassLoader` that sees it. The child loader keeps the
* discovery out of the JVM-wide registries, which the default-build tests assert are empty.
*/
def withServiceFile(service: Class[_], implNames: Seq[String])(f: ClassLoader => Unit): Unit = {
// The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet;
// ensure it before createTempDirectory (which requires its parent to exist).
val baseTmp = new File(System.getProperty("java.io.tmpdir"))
baseTmp.mkdirs()
val svcDir =
Files.createTempDirectory(baseTmp.toPath, s"comet-${service.getSimpleName}-test").toFile
try {
val servicesDir = new File(svcDir, "META-INF/services")
assert(servicesDir.mkdirs(), s"could not create $servicesDir")
Files.write(
new File(servicesDir, service.getName).toPath,
implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8))

f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader))
} finally {
def del(file: File): Unit = {
Option(file.listFiles()).foreach(_.foreach(del))
file.delete()
}
del(svcDir)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,6 @@

package org.apache.comet.rules

import java.io.File
import java.net.URLClassLoader
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.util.ServiceLoader

import scala.annotation.nowarn
Expand All @@ -43,7 +39,7 @@ import org.apache.spark.sql.execution.{FileSourceScanExec, LeafExecNode, SparkPl
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
import org.apache.spark.sql.execution.datasources.v2.BatchScanExec

import org.apache.comet.ContribServices
import org.apache.comet.{ContribServiceFiles, ContribServices}
import org.apache.comet.util.ClassLoaders

/**
Expand Down Expand Up @@ -339,34 +335,8 @@ class CometScanContribSuite extends AnyFunSuite {
}
}

/**
* Writes a `META-INF/services/org.apache.comet.rules.CometScanContrib` naming `impls` into a
* temp directory, loads it through an isolated child `URLClassLoader`, and hands the discovered
* instances to `f`. The child loader keeps the discovery out of the JVM-wide registry that the
* default-build test above asserts is empty.
*/
private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit = {
// The scalatest plugin points java.io.tmpdir at spark/target/tmp, which may not exist yet;
// ensure it before createTempDirectory (which requires its parent to exist).
val baseTmp = new File(System.getProperty("java.io.tmpdir"))
baseTmp.mkdirs()
val svcDir = Files.createTempDirectory(baseTmp.toPath, "comet-scan-contrib-test").toFile
try {
val servicesDir = new File(svcDir, "META-INF/services")
assert(servicesDir.mkdirs(), s"could not create $servicesDir")
Files.write(
new File(servicesDir, classOf[CometScanContrib].getName).toPath,
implNames.mkString("", "\n", "\n").getBytes(StandardCharsets.UTF_8))

f(new URLClassLoader(Array(svcDir.toURI.toURL), getClass.getClassLoader))
} finally {
def del(file: File): Unit = {
Option(file.listFiles()).foreach(_.foreach(del))
file.delete()
}
del(svcDir)
}
}
private def withServiceFile(implNames: Seq[String])(f: ClassLoader => Unit): Unit =
ContribServiceFiles.withServiceFile(classOf[CometScanContrib], implNames)(f)
}

/** Inert leaf plan standing in for whatever node a real contrib would return. */
Expand Down
Loading