From f34086600f8e4e6f01bce1bd9c0957856a6cb802 Mon Sep 17 00:00:00 2001 From: dajiaohuang Date: Sat, 31 Oct 2026 02:20:12 +0800 Subject: [PATCH] fix: respect SQL execution config when captured keys are unset --- .../spark/sql/comet/CapturedSQLConf.scala | 2 + .../comet/CometSessionExtensionsSuite.scala | 49 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CapturedSQLConf.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CapturedSQLConf.scala index 1d15fde5f12..792f80ca821 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CapturedSQLConf.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CapturedSQLConf.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.comet import org.apache.spark.TaskContext +import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.internal.{ReadOnlySQLConf, SQLConf} import org.apache.comet.CometConf @@ -45,6 +46,7 @@ class CapturedSQLConf private (configs: Map[String, String]) extends Serializabl def withConf[T](f: => T): T = { val context = TaskContext.get() if (configs.isEmpty || context == null || !SQLConf.get.isInstanceOf[ReadOnlySQLConf] || + context.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) != null || configs.keys.exists(context.getLocalProperty(_) != null)) { f } else { diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala index 49a2f83b6e7..d6da464804b 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometSessionExtensionsSuite.scala @@ -20,12 +20,61 @@ package org.apache.spark.sql.comet import org.apache.spark.sql.{CometTestBase, SparkSessionExtensions} +import org.apache.spark.sql.execution.SQLExecution +import org.apache.spark.sql.internal.SQLConf import org.apache.comet.CometSparkSessionExtensions import org.apache.comet.rules.CometRule class CometSessionExtensionsSuite extends CometTestBase { + test("captured configs do not override a SQL execution after a setting is unset") { + val staleKey = "spark.comet.test.captured.stale" + val currentKey = "spark.comet.test.captured.current" + val captured = SQLConf.withExistingConf(new SQLConf()) { + SQLConf.get.setConfString(staleKey, "stale") + CapturedSQLConf.capture() + } + withSQLConf(currentKey -> "current") { + spark.conf.unset(staleKey) + def readTaskValues(): Seq[(String, String)] = spark.sparkContext + .parallelize(Seq(1), 1) + .map { _ => + captured.withConf { + val conf = SQLConf.get + (conf.getConfString(staleKey, "missing"), conf.getConfString(currentKey, "missing")) + } + } + .collect() + .toSeq + + // Outside a SQL execution Spark does not propagate these settings, so the captured + // settings still fill the gap as before. + assert(readTaskValues() == Seq(("stale", "missing"))) + // A task property without a SQL execution id still wins over the capture. + val previous = spark.sparkContext.getLocalProperty(staleKey) + try { + spark.sparkContext.setLocalProperty(staleKey, "propagated") + assert(readTaskValues() == Seq(("propagated", "missing"))) + } finally { + spark.sparkContext.setLocalProperty(staleKey, previous) + } + val query = spark.range(1) + val actual = SQLExecution.withNewExecutionId(query.queryExecution) { + readTaskValues() + } + assert(actual == Seq(("missing", "current"))) + + // When the originally captured key is still set, prefer the current value as before. + withSQLConf(staleKey -> "new") { + val current = SQLExecution.withNewExecutionId(query.queryExecution) { + readTaskValues() + } + assert(current == Seq(("new", "current"))) + } + } + } + private def countCometRules(extensions: SparkSessionExtensions): (Int, Int) = { val columnar = extensions .buildColumnarRules(spark)