diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala index f9e93533275..3277c4eb32e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala @@ -19,6 +19,7 @@ package org.apache.spark.sql.comet.execution.shuffle +import java.io.File import java.nio.file.{Files, Paths} import java.util.concurrent.{ScheduledFuture, TimeUnit} import java.util.concurrent.atomic.AtomicBoolean @@ -73,6 +74,9 @@ class CometNativeShuffleWriter[K, V]( // var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the // tests that read it, is the only thing that ever touches this. private[shuffle] var mapStatus: MapStatus = _ + // Temp data file for this local map attempt. Cleared after a successful commit so + // stop(true) does not delete the committed data file. + private[shuffle] var localTempData: Option[File] = None private var stopped = false private lazy val effectivePartitionCount = remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions) @@ -107,6 +111,7 @@ class CometNativeShuffleWriter[K, V]( destination.onSizeLimitExceeded(failure) } } + discardLocalTemp(Some(failure)) throw failure } } @@ -116,9 +121,9 @@ class CometNativeShuffleWriter[K, V]( val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver] val dataFile = resolver.getDataFile(shuffleId, mapId) - Some( - CometNativeShuffleWriter - .LocalShuffleOutput(resolver, dataFile.getPath.replace(".data", ".data.tmp"))) + val tempDataFile = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile) + localTempData = Some(tempDataFile) + Some(CometNativeShuffleWriter.LocalShuffleOutput(resolver, tempDataFile.getPath)) } else { None } @@ -274,6 +279,7 @@ class CometNativeShuffleWriter[K, V]( partitionLengths, Array.empty, tempDataFilePath.toFile) + localTempData = None mapStatus = MapStatus.apply(SparkEnv.get.blockManager.shuffleServerId, partitionLengths, mapId) } @@ -485,7 +491,13 @@ class CometNativeShuffleWriter[K, V]( override def stop(success: Boolean): Option[MapStatus] = { remoteDestination match { - case None => if (success) Some(mapStatus) else None + case None => + if (success) { + Some(mapStatus) + } else { + discardLocalTemp(None) + None + } case Some(destination) => synchronized { if (stopped) { @@ -512,10 +524,35 @@ class CometNativeShuffleWriter[K, V]( } override def getPartitionLengths(): Array[Long] = partitionLengths + + private def discardLocalTemp(failure: Option[Throwable]): Unit = { + val temp = localTempData + localTempData = None + temp.foreach { file => + try { + Files.deleteIfExists(file.toPath) + } catch { + case cleanup: Throwable => + failure match { + case Some(original) => original.addSuppressed(cleanup) + case None => + logWarning("Could not delete the native shuffle temp data file", cleanup) + } + } + } + } } private[shuffle] object CometNativeShuffleWriter { + // Spark appends a unique suffix in this directory. Replacing `.data` would also + // rewrite a directory whose name contains `.data`. + private[shuffle] def localTempDataFile( + resolver: IndexShuffleBlockResolver, + dataFile: File): File = { + resolver.createTempFile(dataFile) + } + /** * Declared here rather than inside the class: as an inner case class every type test against it * carries an outer reference that cannot be checked at run time, which `-Xlint` reports. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala index 8e72e37384a..f298bdc3234 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriterSuite.scala @@ -19,7 +19,10 @@ package org.apache.spark.sql.comet.execution.shuffle -import org.apache.spark.TaskContext +import java.io.File + +import org.apache.spark.{SparkEnv, TaskContext} +import org.apache.spark.shuffle.IndexShuffleBlockResolver import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.vectorized.ColumnarBatch @@ -76,4 +79,35 @@ class CometNativeShuffleWriterSuite extends CometTestBase { } } } + + test("native shuffle temp file stays in a local dir named local.data") { + withTempDir { dir => + val base = new File(dir, "local.data") + assert(base.mkdir()) + val dataFile = new File(base, "shuffle_0_3_0.data") + val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver + .asInstanceOf[IndexShuffleBlockResolver] + val temp = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile) + assert(temp.getParentFile.getName == "local.data") + assert(!temp.getPath.contains("local.data.tmp")) + } + } + + test("native shuffle stop(false) deletes a tracked temp file") { + withTempDir { dir => + val failed = new File(dir, "shuffle_0_0_0.data.failed") + assert(failed.createNewFile()) + val writer = new CometNativeShuffleWriter[Int, ColumnarBatch](null, null, null, null, 0, 0, + 0L, null, null) + writer.localTempData = Some(failed) + assert(writer.stop(false).isEmpty) + assert(!failed.exists()) + + val committed = new File(dir, "shuffle_0_0_0.data.committed") + assert(committed.createNewFile()) + writer.localTempData = Some(committed) + assert(writer.stop(true).isDefined) + assert(committed.exists()) + } + } }