From 9f060f88ea42eb7c56dd422c59fe42cef568522d Mon Sep 17 00:00:00 2001 From: Po-Han Shih Date: Mon, 16 Nov 2026 13:15:48 +0800 Subject: [PATCH 1/3] test: fail when a local.data directory rewrites the shuffle temp path String.replace turns every .data into .data.tmp, so a spark.local.dir named local.data is rewritten before the native writer opens the file. --- .../shuffle/CometNativeShuffleWriter.scala | 14 +++++++++++--- .../CometNativeShuffleWriterSuite.scala | 18 +++++++++++++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index f9e93533275..0952b260d37 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.comet.execution.shuffle +import java.io.File import java.nio.file.{Files, Paths} import java.util.concurrent.{ScheduledFuture, TimeUnit} import java.util.concurrent.atomic.AtomicBoolean @@ -116,9 +117,8 @@ class CometNativeShuffleWriter[K, V]( val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver] val dataFile = resolver.getDataFile(shuffleId, mapId) - Some( - CometNativeShuffleWriter - .LocalShuffleOutput(resolver, dataFile.getPath.replace(".data", ".data.tmp"))) + val tempDataFile = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile) + Some(CometNativeShuffleWriter.LocalShuffleOutput(resolver, tempDataFile.getPath)) } else { None } @@ -516,6 +516,14 @@ class CometNativeShuffleWriter[K, V]( private[shuffle] object CometNativeShuffleWriter { + // A directory named `local.data` must stay intact. `String.replace` rewrites every `.data`. + private[shuffle] def localTempDataFile( + resolver: IndexShuffleBlockResolver, + dataFile: File): File = { + require(resolver != null) + new File(dataFile.getPath.replace(".data", ".data.tmp")) + } + /** * Declared here rather than inside the class: as an inner case class every type test against it * carries an outer reference that cannot be checked at run time, which `-Xlint` reports. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 8e72e37384a..29b2cc3c204 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -19,7 +19,10 @@ package org.apache.spark.sql.comet.execution.shuffle -import org.apache.spark.TaskContext +import java.io.File + +import org.apache.spark.{SparkEnv, TaskContext} +import org.apache.spark.shuffle.IndexShuffleBlockResolver import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.vectorized.ColumnarBatch @@ -76,4 +79,17 @@ class CometNativeShuffleWriterSuite extends CometTestBase { } } } + + test("native shuffle temp file stays in a local dir named local.data") { + withTempDir { dir => + val base = new File(dir, "local.data") + assert(base.mkdir()) + val dataFile = new File(base, "shuffle_0_3_0.data") + val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver + .asInstanceOf[IndexShuffleBlockResolver] + val temp = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile) + assert(temp.getParentFile.getName == "local.data") + assert(!temp.getPath.contains("local.data.tmp")) + } + } } From 106b66b5866016523dc6cb2084dfb59c382d1e8f Mon Sep 17 00:00:00 2001 From: Po-Han Shih Date: Mon, 16 Nov 2026 13:21:35 +0800 Subject: [PATCH 2/3] fix: keep native shuffle temp files beside the data file IndexShuffleBlockResolver.createTempFile appends a unique suffix in the same directory. A failed map attempt deletes that temp file, and a successful commit still leaves the renamed data file in place. --- .../shuffle/CometNativeShuffleWriter.scala | 37 +++++++++++++++++-- .../CometNativeShuffleWriterSuite.scala | 26 +++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index 0952b260d37..3277c4eb32e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -74,6 +74,9 @@ class CometNativeShuffleWriter[K, V]( // var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the // tests that read it, is the only thing that ever touches this. private[shuffle] var mapStatus: MapStatus = _ + // Temp data file for this local map attempt. Cleared after a successful commit so + // stop(true) does not delete the committed data file. + private[shuffle] var localTempData: Option[File] = None private var stopped = false private lazy val effectivePartitionCount = remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions) @@ -108,6 +111,7 @@ class CometNativeShuffleWriter[K, V]( destination.onSizeLimitExceeded(failure) } } + discardLocalTemp(Some(failure)) throw failure } } @@ -118,6 +122,7 @@ class CometNativeShuffleWriter[K, V]( SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver] val dataFile = resolver.getDataFile(shuffleId, mapId) val tempDataFile = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile) + localTempData = Some(tempDataFile) Some(CometNativeShuffleWriter.LocalShuffleOutput(resolver, tempDataFile.getPath)) } else { None @@ -274,6 +279,7 @@ class CometNativeShuffleWriter[K, V]( partitionLengths, Array.empty, tempDataFilePath.toFile) + localTempData = None mapStatus = MapStatus.apply(SparkEnv.get.blockManager.shuffleServerId, partitionLengths, mapId) } @@ -485,7 +491,13 @@ class CometNativeShuffleWriter[K, V]( override def stop(success: Boolean): Option[MapStatus] = { remoteDestination match { - case None => if (success) Some(mapStatus) else None + case None => + if (success) { + Some(mapStatus) + } else { + discardLocalTemp(None) + None + } case Some(destination) => synchronized { if (stopped) { @@ -512,16 +524,33 @@ class CometNativeShuffleWriter[K, V]( } override def getPartitionLengths(): Array[Long] = partitionLengths + + private def discardLocalTemp(failure: Option[Throwable]): Unit = { + val temp = localTempData + localTempData = None + temp.foreach { file => + try { + Files.deleteIfExists(file.toPath) + } catch { + case cleanup: Throwable => + failure match { + case Some(original) => original.addSuppressed(cleanup) + case None => + logWarning("Could not delete the native shuffle temp data file", cleanup) + } + } + } + } } private[shuffle] object CometNativeShuffleWriter { - // A directory named `local.data` must stay intact. `String.replace` rewrites every `.data`. + // Spark appends a unique suffix in this directory. Replacing `.data` would also + // rewrite a directory whose name contains `.data`. private[shuffle] def localTempDataFile( resolver: IndexShuffleBlockResolver, dataFile: File): File = { - require(resolver != null) - new File(dataFile.getPath.replace(".data", ".data.tmp")) + resolver.createTempFile(dataFile) } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 29b2cc3c204..55244f6ca7a 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -92,4 +92,30 @@ class CometNativeShuffleWriterSuite extends CometTestBase { assert(!temp.getPath.contains("local.data.tmp")) } } + + test("native shuffle stop(false) deletes a tracked temp file") { + withTempDir { dir => + val failed = new File(dir, "shuffle_0_0_0.data.failed") + assert(failed.createNewFile()) + val writer = new CometNativeShuffleWriter[Int, ColumnarBatch]( + null, + null, + null, + null, + 0, + 0, + 0L, + null, + null) + writer.localTempData = Some(failed) + assert(writer.stop(false).isEmpty) + assert(!failed.exists()) + + val committed = new File(dir, "shuffle_0_0_0.data.committed") + assert(committed.createNewFile()) + writer.localTempData = Some(committed) + assert(writer.stop(true).isDefined) + assert(committed.exists()) + } + } } From ea6a24bf3494d552ee7c1c151864be5f068e93f7 Mon Sep 17 00:00:00 2001 From: Po-Han Shih Date: Tue, 17 Nov 2026 02:40:49 +0800 Subject: [PATCH 3/3] style: format the native shuffle temp file test Spotless on Linux rejects the stop(false) writer constructor. The apply used the real Apache header and only this file. --- .../shuffle/CometNativeShuffleWriterSuite.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 55244f6ca7a..f298bdc3234 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -97,16 +97,8 @@ class CometNativeShuffleWriterSuite extends CometTestBase { withTempDir { dir => val failed = new File(dir, "shuffle_0_0_0.data.failed") assert(failed.createNewFile()) - val writer = new CometNativeShuffleWriter[Int, ColumnarBatch]( - null, - null, - null, - null, - 0, - 0, - 0L, - null, - null) + val writer = new CometNativeShuffleWriter[Int, ColumnarBatch](null, null, null, null, 0, 0, + 0L, null, null) writer.localTempData = Some(failed) assert(writer.stop(false).isEmpty) assert(!failed.exists())