From 38feaade7c89e5aa4cc25dc2cae3ef478be0c4cf Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 14 Oct 2026 16:46:23 -0600 Subject: [PATCH 1/4] build: remove Guava dependency --- .../adding_a_new_operator.md | 8 +-- pom.xml | 20 ------- spark/pom.xml | 31 ----------- .../CometBypassMergeSortShuffleWriter.java | 35 +++--------- .../shuffle/CometDiskBlockWriter.java | 4 +- .../shuffle/CometUnsafeShuffleWriter.java | 39 ++++++++----- .../spark/sql/comet/CometBatchScanExec.scala | 6 +- .../comet/CometBroadcastExchangeExec.scala | 5 +- .../spark/sql/comet/CometCoalesceExec.scala | 6 +- .../sql/comet/CometCollectLimitExec.scala | 6 +- .../sql/comet/CometCsvNativeScanExec.scala | 6 +- .../comet/CometIcebergNativeScanExec.scala | 8 +-- .../sql/comet/CometLocalTableScanExec.scala | 6 +- .../spark/sql/comet/CometLocalTopKExec.scala | 6 +- .../spark/sql/comet/CometNativeScanExec.scala | 6 +- .../spark/sql/comet/CometRangeExec.scala | 6 +- .../spark/sql/comet/CometSampleExec.scala | 6 +- .../spark/sql/comet/CometWindowExec.scala | 6 +- .../sql/comet/CometWindowGroupLimitExec.scala | 6 +- .../shuffle/CometShuffleExchangeExec.scala | 5 +- .../apache/spark/sql/comet/operators.scala | 30 +++++----- .../comet/parquet/ParquetReadSuite.scala | 5 +- .../sql/comet/CometPlanEqualitySuite.scala | 55 ++++++++++++++++--- 23 files changed, 141 insertions(+), 170 deletions(-) diff --git a/docs/source/contributor-guide/adding_a_new_operator.md b/docs/source/contributor-guide/adding_a_new_operator.md index 15c4e9169b6..88272332882 100644 --- a/docs/source/contributor-guide/adding_a_new_operator.md +++ b/docs/source/contributor-guide/adding_a_new_operator.md @@ -226,7 +226,7 @@ The validation workflow in `CometExecRule.isOperatorEnabled`: #### Simple Example (Filter) ```scala -import com.google.common.base.Objects +import java.util.Objects object CometFilterExec extends CometOperatorSerde[FilterExec] { @@ -284,14 +284,14 @@ case class CometFilterExec( } } - override def hashCode(): Int = Objects.hashCode(output, condition, child) + override def hashCode(): Int = Objects.hash(output, condition, child) } ``` #### More Complex Example (Project) ```scala -import com.google.common.base.Objects +import java.util.Objects object CometProjectExec extends CometOperatorSerde[ProjectExec] { @@ -348,7 +348,7 @@ case class CometProjectExec( } } - override def hashCode(): Int = Objects.hashCode(output, projectList, child) + override def hashCode(): Int = Objects.hash(output, projectList, child) override protected def outputExpressions: Seq[NamedExpression] = projectList } diff --git a/pom.xml b/pom.xml index a020f0719f9..279bbd67aa7 100644 --- a/pom.xml +++ b/pom.xml @@ -92,7 +92,6 @@ under the License. release that exists for each of their Scala versions (2.13.16, 2.13.17 and 2.13.18). --> 4.13.10 2.0.17 - 33.2.1-jre 1.21.4 2.31.51 - - com.google.guava - guava - ${guava.version} - - - org.apache.spark @@ -1276,15 +1265,6 @@ under the License. javax.annotation.meta.TypeQualifierNickname - - com.google.guava - guava - - com.google.thirdparty.publicsuffix.TrieParser - com.google.thirdparty.publicsuffix.PublicSuffixPatterns - com.google.thirdparty.publicsuffix.PublicSuffixType - - org.apache.datasketches datasketches-memory diff --git a/spark/pom.xml b/spark/pom.xml index 18c0ac60914..1a61581fca1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -123,28 +123,6 @@ under the License. junit test - - com.google.guava - guava - - - * - * - - - - - - com.google.guava - failureaccess - 1.0.2 - test - org.codehaus.jackson jackson-mapper-asl @@ -720,7 +698,6 @@ under the License. org.apache.arrow:* com.google.protobuf:protobuf-java - com.google.guava:guava @@ -780,14 +757,6 @@ under the License. com.google.protobuf ${comet.shade.packageName}.protobuf - - com.google.common - ${comet.shade.packageName}.guava - - - com.google.thirdparty - ${comet.shade.packageName}.guava.thirdparty - ${project.basedir}/../../native/target/debug false + + 33.2.1-jre diff --git a/spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java b/spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java index 3ab34fe0496..4bf7e9c0aec 100644 --- a/spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java +++ b/spark/src/main/java/org/apache/comet/udf/CometUdfBridge.java @@ -32,8 +32,6 @@ import org.apache.spark.comet.CometTaskContextShim; import org.apache.spark.util.TaskCompletionListener; -import com.google.common.annotations.VisibleForTesting; - import org.apache.comet.util.ClassLoaders; /** @@ -315,7 +313,6 @@ public static void releasePlan(long taskAttemptId, long planId) { } /** The instance of {@code udfClassName} that task {@code taskAttemptId} holds, or null. */ - @VisibleForTesting public static CometUDF instanceFor(long taskAttemptId, String udfClassName) { ConcurrentHashMap perTask = INSTANCES.get(taskAttemptId); return perTask == null ? null : perTask.get(udfClassName); diff --git a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala index 20646219625..4f0f8d907c7 100644 --- a/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala +++ b/spark/src/test/scala/org/apache/comet/parquet/ParquetReadSuite.scala @@ -1407,7 +1407,8 @@ abstract class ParquetReadSuite extends CometTestBase { checkAnswer( df, (0 until 10).map(n => - Row(new BigDecimal(BigInteger.valueOf(n).add(BigInteger.valueOf(Long.MaxValue)))))) + Row(new BigDecimal( + BigInteger.valueOf(n.toLong).add(BigInteger.valueOf(Long.MaxValue)))))) } } } From 12275be605a40c2aa3b84f050d38c57f4f6cdbad Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 15 Oct 2026 17:17:35 -0600 Subject: [PATCH 3/4] Add Guava to Spark test runtime --- spark/pom.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/spark/pom.xml b/spark/pom.xml index ebd43c9fc4c..b1035c5fd7e 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -36,6 +36,9 @@ under the License. false + + 33.2.1-jre @@ -123,6 +126,12 @@ under the License. junit test + + com.google.guava + guava + ${guava.version} + test + org.codehaus.jackson jackson-mapper-asl From 3b05646079ac3a755432864ddbd04b3b1386375b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 09:27:44 -0700 Subject: [PATCH 4/4] build: ban Guava outside test scope, and drop it from CometTextNativeScanExec CometTextNativeScanExec, added on main since the last merge, used Guava's Objects.hashCode. It uses java.util.Objects.hash now, as the other operators do. A new enforcer execution, ban-guava-outside-tests, fails the build when guava or failureaccess reaches the compile or runtime scope, directly or transitively, so that Comet does not start shipping Guava again. Co-Authored-By: Claude Opus 5.5 --- pom.xml | 23 +++++++++++++++++++ .../sql/comet/CometTextNativeScanExec.scala | 6 ++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index f54beab4ea4..ad1189269d5 100644 --- a/pom.xml +++ b/pom.xml @@ -1373,6 +1373,29 @@ under the License. + + + ban-guava-outside-tests + + enforce + + + + + true + + com.google.guava:guava:*:*:compile + com.google.guava:guava:*:*:runtime + com.google.guava:failureaccess:*:*:compile + com.google.guava:failureaccess:*:*:runtime + + Comet does not ship Guava. Use the JDK equivalent instead, or keep the + dependency at test scope. + + + + no-duplicate-declared-dependencies diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometTextNativeScanExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometTextNativeScanExec.scala index 7ee18b56703..b8eb5c308e4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometTextNativeScanExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometTextNativeScanExec.scala @@ -19,6 +19,8 @@ package org.apache.spark.sql.comet +import java.util.Objects + import scala.jdk.CollectionConverters._ import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} @@ -30,8 +32,6 @@ import org.apache.spark.sql.execution.datasources.v2.BatchScanExec import org.apache.spark.sql.execution.datasources.v2.text.TextScan import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import com.google.common.base.Objects - import org.apache.comet.{CometConf, ConfigEntry} import org.apache.comet.CometSparkSessionExtensions.withFallbackReason import org.apache.comet.serde.{CometOperatorSerde, OperatorOuterClass} @@ -92,7 +92,7 @@ case class CometTextNativeScanExec( } override def hashCode(): Int = { - Objects.hashCode(output, serializedPlanOpt, originalPlan) + Objects.hash(output, serializedPlanOpt, originalPlan) } }