Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.apache.spark.sql.comet.execution.shuffle

import java.io.File
import java.nio.file.{Files, Paths}
import java.util.concurrent.{ScheduledFuture, TimeUnit}
import java.util.concurrent.atomic.AtomicBoolean
Expand Down Expand Up @@ -73,6 +74,9 @@ class CometNativeShuffleWriter[K, V](
// var exposing it trips `-Xlint:inaccessible`. The shuffle package, which includes the
// tests that read it, is the only thing that ever touches this.
private[shuffle] var mapStatus: MapStatus = _
// Temp data file for this local map attempt. Cleared after a successful commit so
// stop(true) does not delete the committed data file.
private[shuffle] var localTempData: Option[File] = None
private var stopped = false
private lazy val effectivePartitionCount =
remoteDestination.map(_.numPartitions).getOrElse(outputPartitioning.numPartitions)
Expand Down Expand Up @@ -107,6 +111,7 @@ class CometNativeShuffleWriter[K, V](
destination.onSizeLimitExceeded(failure)
}
}
discardLocalTemp(Some(failure))
throw failure
}
}
Expand All @@ -116,9 +121,9 @@ class CometNativeShuffleWriter[K, V](
val resolver =
SparkEnv.get.shuffleManager.shuffleBlockResolver.asInstanceOf[IndexShuffleBlockResolver]
val dataFile = resolver.getDataFile(shuffleId, mapId)
Some(
CometNativeShuffleWriter
.LocalShuffleOutput(resolver, dataFile.getPath.replace(".data", ".data.tmp")))
val tempDataFile = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile)
localTempData = Some(tempDataFile)
Some(CometNativeShuffleWriter.LocalShuffleOutput(resolver, tempDataFile.getPath))
} else {
None
}
Expand Down Expand Up @@ -274,6 +279,7 @@ class CometNativeShuffleWriter[K, V](
partitionLengths,
Array.empty,
tempDataFilePath.toFile)
localTempData = None
mapStatus =
MapStatus.apply(SparkEnv.get.blockManager.shuffleServerId, partitionLengths, mapId)
}
Expand Down Expand Up @@ -485,7 +491,13 @@ class CometNativeShuffleWriter[K, V](

override def stop(success: Boolean): Option[MapStatus] = {
remoteDestination match {
case None => if (success) Some(mapStatus) else None
case None =>
if (success) {
Some(mapStatus)
} else {
discardLocalTemp(None)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This discardLocalTemp(None) repeats the cleanup in the catch of write (line 114). Spark's ShuffleWriteProcessor calls stop(false) only after write threw an Exception, and the catch here already handles every Throwable. localTempData is only set inside writeInternal, so I could not find a path where stop(false) sees a temp file that write did not already delete. CometBypassMergeSortShuffleWriter follows the same rule and cleans up in write only. Dropping this branch and the stop(false) test would leave one cleanup path.

None
}
case Some(destination) =>
synchronized {
if (stopped) {
Expand All @@ -512,10 +524,35 @@ class CometNativeShuffleWriter[K, V](
}

override def getPartitionLengths(): Array[Long] = partitionLengths

private def discardLocalTemp(failure: Option[Throwable]): Unit = {
val temp = localTempData
localTempData = None
temp.foreach { file =>
try {
Files.deleteIfExists(file.toPath)
} catch {
case cleanup: Throwable =>
failure match {
case Some(original) => original.addSuppressed(cleanup)
case None =>
logWarning("Could not delete the native shuffle temp data file", cleanup)
}
}
}
}
}

private[shuffle] object CometNativeShuffleWriter {

// Spark appends a unique suffix in this directory. Replacing `.data` would also
// rewrite a directory whose name contains `.data`.
private[shuffle] def localTempDataFile(
resolver: IndexShuffleBlockResolver,
dataFile: File): File = {
resolver.createTempFile(dataFile)
}

/**
* Declared here rather than inside the class: as an inner case class every type test against it
* carries an outer reference that cannot be checked at run time, which `-Xlint` reports.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@

package org.apache.spark.sql.comet.execution.shuffle

import org.apache.spark.TaskContext
import java.io.File

import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.shuffle.IndexShuffleBlockResolver
import org.apache.spark.sql.CometTestBase
import org.apache.spark.sql.vectorized.ColumnarBatch

Expand Down Expand Up @@ -76,4 +79,35 @@ class CometNativeShuffleWriterSuite extends CometTestBase {
}
}
}

test("native shuffle temp file stays in a local dir named local.data") {
withTempDir { dir =>
val base = new File(dir, "local.data")
assert(base.mkdir())
val dataFile = new File(base, "shuffle_0_3_0.data")
val resolver = SparkEnv.get.shuffleManager.shuffleBlockResolver
.asInstanceOf[IndexShuffleBlockResolver]
val temp = CometNativeShuffleWriter.localTempDataFile(resolver, dataFile)
assert(temp.getParentFile.getName == "local.data")
assert(!temp.getPath.contains("local.data.tmp"))
}
}

test("native shuffle stop(false) deletes a tracked temp file") {
withTempDir { dir =>
val failed = new File(dir, "shuffle_0_0_0.data.failed")
assert(failed.createNewFile())
val writer = new CometNativeShuffleWriter[Int, ColumnarBatch](null, null, null, null, 0, 0,
0L, null, null)
writer.localTempData = Some(failed)
assert(writer.stop(false).isEmpty)
assert(!failed.exists())

val committed = new File(dir, "shuffle_0_0_0.data.committed")
assert(committed.createNewFile())
writer.localTempData = Some(committed)
assert(writer.stop(true).isDefined)
assert(committed.exists())
}
}
}