From 10486302c1131b138547ebf6534c1517fe88113d Mon Sep 17 00:00:00 2001 From: Wu Shuwen Date: Sat, 14 Nov 2026 20:10:17 +0000 Subject: [PATCH] fix: preserve session settings in JVM shuffle writers (#7526) * fix: propagate session settings to JVM shuffle writers * Reuse native shuffle configuration snapshot (cherry picked from commit 55ea0d4f770b20841ac5514c0b9990d83bb9fbf3) --- .../shuffle/CometShuffleDependency.scala | 4 ++ .../shuffle/CometShuffleManager.scala | 61 ++++++++++++------- .../exec/CometColumnarShuffleSuite.scala | 39 ++++++++++++ 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala index ff813430ed9..dadc4440323 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleDependency.scala @@ -100,6 +100,10 @@ class CometShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag]( mapSideCombine, shuffleWriterProcessor) { + /** The Comet settings used by JVM shuffle writers on executor task threads. */ + private[shuffle] val writerSQLConf: CapturedSQLConf = + nativeShuffleSpec.map(_.sqlConf).getOrElse(CapturedSQLConf.capture()) + @transient @volatile private var materializationInstance : CometCelebornShuffleMaterialization[K, V, C] = _ diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala index 4cf42c5d312..ce12111dc77 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleManager.scala @@ -32,7 +32,7 @@ import org.apache.spark.internal.{config, Logging} import org.apache.spark.shuffle._ import org.apache.spark.shuffle.api.ShuffleExecutorComponents import org.apache.spark.shuffle.sort.{BypassMergeSortShuffleHandle, SerializedShuffleHandle, SortShuffleManager, SortShuffleWriter} -import org.apache.spark.sql.comet.PlanDataInjector +import org.apache.spark.sql.comet.{CapturedSQLConf, PlanDataInjector} import org.apache.spark.sql.internal.SQLConf import org.apache.spark.util.collection.OpenHashSet @@ -218,6 +218,21 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { } } + /** Apply the driver capture to writer construction and each executor-side operation. */ + private def withWriterConf[K, V](sqlConf: CapturedSQLConf)( + create: => ShuffleWriter[K, V]): ShuffleWriter[K, V] = { + val writer = sqlConf.withConf(create) + new ShuffleWriter[K, V] { + override def write(records: Iterator[Product2[K, V]]): Unit = + sqlConf.withConf(writer.write(records)) + + override def stop(success: Boolean): Option[org.apache.spark.scheduler.MapStatus] = + sqlConf.withConf(writer.stop(success)) + + override def getPartitionLengths(): Array[Long] = writer.getPartitionLengths() + } + } + /** Get a writer for a given partition. Called on executors by map tasks. */ @nowarn("msg=references private") override def getWriter[K, V]( @@ -249,30 +264,34 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging { val bypassDep = bypassMergeSortHandle.dependency.asInstanceOf[CometShuffleDependency[_, _, _]] val bypassEncodeMetric = bypassDep.shuffleWriteMetrics.get("encode_time").orNull - new CometBypassMergeSortShuffleWriter( - env.blockManager, - context.taskMemoryManager(), - context, - bypassMergeSortHandle, - mapId, - env.conf, - metrics, - shuffleExecutorComponents, - bypassEncodeMetric) + withWriterConf(bypassDep.writerSQLConf) { + new CometBypassMergeSortShuffleWriter( + env.blockManager, + context.taskMemoryManager(), + context, + bypassMergeSortHandle, + mapId, + env.conf, + metrics, + shuffleExecutorComponents, + bypassEncodeMetric) + } case unsafeShuffleHandle: CometSerializedShuffleHandle[K @unchecked, V @unchecked] => val unsafeDep = unsafeShuffleHandle.dependency.asInstanceOf[CometShuffleDependency[_, _, _]] val unsafeEncodeMetric = unsafeDep.shuffleWriteMetrics.get("encode_time").orNull - new CometUnsafeShuffleWriter( - env.blockManager, - context.taskMemoryManager(), - unsafeShuffleHandle, - mapId, - context, - env.conf, - metrics, - shuffleExecutorComponents, - unsafeEncodeMetric) + withWriterConf(unsafeDep.writerSQLConf) { + new CometUnsafeShuffleWriter( + env.blockManager, + context.taskMemoryManager(), + unsafeShuffleHandle, + mapId, + context, + env.conf, + metrics, + shuffleExecutorComponents, + unsafeEncodeMetric) + } case _ => // It is a Spark shuffle dependency, so we use Spark Sort Shuffle Writer. sortShuffleManager.getWriter(handle, mapId, context, metrics) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala index 01a5e31daa5..8129a321da7 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometColumnarShuffleSuite.scala @@ -68,6 +68,45 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar setupTestData() + test("JVM shuffle writers see session configs outside SQL execution") { + withParquetTable((0 until 100).map(i => (i % 10, i)), "writer_conf") { + Seq("0", "10000").foreach { maxWriters => + withSQLConf( + CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.key -> maxWriters, + CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "bogus") { + val df = spark.table("writer_conf").repartition(2, col("_1")) + val exchange = checkCometExchange(df, 1, false).head + val expectedHandle = if (maxWriters == "0") { + "CometSerializedShuffleHandle" + } else { + "CometBypassMergeSortShuffleHandle" + } + assert( + exchange.shuffleDependency.shuffleHandle.getClass.getName.endsWith(expectedHandle)) + val error = intercept[Exception] { + df.queryExecution.toRdd.count() + } + assert(error.toString.contains("spark.comet.shuffle.compression.codec"), error) + val sqlError = intercept[Exception] { + df.collect() + } + assert(sqlError.toString.contains("spark.comet.shuffle.compression.codec"), sqlError) + } + withSQLConf( + CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.key -> maxWriters, + CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "zstd") { + assert( + spark + .table("writer_conf") + .repartition(2, col("_1")) + .queryExecution + .toRdd + .count() == 100) + } + } + } + } + test("Fallback to Spark when shuffling on struct with duplicate field name") { val df = sql(""" | SELECT max(struct(a, record.*, b)) as r FROM