Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ class CometShuffleDependency[K: ClassTag, V: ClassTag, C: ClassTag](
mapSideCombine,
shuffleWriterProcessor) {

/** The Comet settings used by JVM shuffle writers on executor task threads. */
private[shuffle] val writerSQLConf: CapturedSQLConf =
nativeShuffleSpec.map(_.sqlConf).getOrElse(CapturedSQLConf.capture())

@transient @volatile private var materializationInstance
: CometCelebornShuffleMaterialization[K, V, C] = _

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import org.apache.spark.internal.{config, Logging}
import org.apache.spark.shuffle._
import org.apache.spark.shuffle.api.ShuffleExecutorComponents
import org.apache.spark.shuffle.sort.{BypassMergeSortShuffleHandle, SerializedShuffleHandle, SortShuffleManager, SortShuffleWriter}
import org.apache.spark.sql.comet.PlanDataInjector
import org.apache.spark.sql.comet.{CapturedSQLConf, PlanDataInjector}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.util.collection.OpenHashSet

Expand Down Expand Up @@ -218,6 +218,21 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging {
}
}

/** Apply the driver capture to writer construction and each executor-side operation. */
private def withWriterConf[K, V](sqlConf: CapturedSQLConf)(
create: => ShuffleWriter[K, V]): ShuffleWriter[K, V] = {
val writer = sqlConf.withConf(create)
new ShuffleWriter[K, V] {
override def write(records: Iterator[Product2[K, V]]): Unit =
sqlConf.withConf(writer.write(records))

override def stop(success: Boolean): Option[org.apache.spark.scheduler.MapStatus] =
sqlConf.withConf(writer.stop(success))

override def getPartitionLengths(): Array[Long] = writer.getPartitionLengths()
}
}

/** Get a writer for a given partition. Called on executors by map tasks. */
@nowarn("msg=references private")
override def getWriter[K, V](
Expand Down Expand Up @@ -249,30 +264,34 @@ class CometShuffleManager(conf: SparkConf) extends ShuffleManager with Logging {
val bypassDep =
bypassMergeSortHandle.dependency.asInstanceOf[CometShuffleDependency[_, _, _]]
val bypassEncodeMetric = bypassDep.shuffleWriteMetrics.get("encode_time").orNull
new CometBypassMergeSortShuffleWriter(
env.blockManager,
context.taskMemoryManager(),
context,
bypassMergeSortHandle,
mapId,
env.conf,
metrics,
shuffleExecutorComponents,
bypassEncodeMetric)
withWriterConf(bypassDep.writerSQLConf) {
new CometBypassMergeSortShuffleWriter(
env.blockManager,
context.taskMemoryManager(),
context,
bypassMergeSortHandle,
mapId,
env.conf,
metrics,
shuffleExecutorComponents,
bypassEncodeMetric)
}
case unsafeShuffleHandle: CometSerializedShuffleHandle[K @unchecked, V @unchecked] =>
val unsafeDep =
unsafeShuffleHandle.dependency.asInstanceOf[CometShuffleDependency[_, _, _]]
val unsafeEncodeMetric = unsafeDep.shuffleWriteMetrics.get("encode_time").orNull
new CometUnsafeShuffleWriter(
env.blockManager,
context.taskMemoryManager(),
unsafeShuffleHandle,
mapId,
context,
env.conf,
metrics,
shuffleExecutorComponents,
unsafeEncodeMetric)
withWriterConf(unsafeDep.writerSQLConf) {
new CometUnsafeShuffleWriter(
env.blockManager,
context.taskMemoryManager(),
unsafeShuffleHandle,
mapId,
context,
env.conf,
metrics,
shuffleExecutorComponents,
unsafeEncodeMetric)
}
case _ =>
// It is a Spark shuffle dependency, so we use Spark Sort Shuffle Writer.
sortShuffleManager.getWriter(handle, mapId, context, metrics)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,45 @@ abstract class CometColumnarShuffleSuite extends CometTestBase with AdaptiveSpar

setupTestData()

test("JVM shuffle writers see session configs outside SQL execution") {
withParquetTable((0 until 100).map(i => (i % 10, i)), "writer_conf") {
Seq("0", "10000").foreach { maxWriters =>
withSQLConf(
CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.key -> maxWriters,
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "bogus") {
val df = spark.table("writer_conf").repartition(2, col("_1"))
val exchange = checkCometExchange(df, 1, false).head
val expectedHandle = if (maxWriters == "0") {
"CometSerializedShuffleHandle"
} else {
"CometBypassMergeSortShuffleHandle"
}
assert(
exchange.shuffleDependency.shuffleHandle.getClass.getName.endsWith(expectedHandle))
val error = intercept[Exception] {
df.queryExecution.toRdd.count()
}
assert(error.toString.contains("spark.comet.shuffle.compression.codec"), error)
val sqlError = intercept[Exception] {
df.collect()
}
assert(sqlError.toString.contains("spark.comet.shuffle.compression.codec"), sqlError)
}
withSQLConf(
CometConf.COMET_SHUFFLE_JVM_MAX_WRITERS_PER_EXECUTOR.key -> maxWriters,
CometConf.COMET_SHUFFLE_COMPRESSION_CODEC.key -> "zstd") {
assert(
spark
.table("writer_conf")
.repartition(2, col("_1"))
.queryExecution
.toRdd
.count() == 100)
}
}
}
}

test("Fallback to Spark when shuffling on struct with duplicate field name") {
val df = sql("""
| SELECT max(struct(a, record.*, b)) as r FROM
Expand Down