diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index c55d0b4378b..d1f6c6cbd10 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -120,7 +120,10 @@ share (`SharedDecode`), and `CometSparkToColumnarExec`. These are real off-heap container RSS that neither Spark's `TaskMemoryManager` nor Comet's native memory pool sees. In practice the volume is modest, a batch at a time per stream, but there is no ceiling and no backpressure. A broadcast is the largest case: a task that reads one holds all of it decoded, and -the tasks of an executor that read it at the same time share one such copy. The executor's memory +the tasks of an executor that read it at the same time share one such copy. The executor keeps that +copy for `spark.comet.exec.broadcastExchange.shareDecoded.idleTimeout` after the last of those +tasks completes, so that the next wave of tasks reads it too, and caps the idle copies of all +broadcasts at `spark.comet.exec.broadcastExchange.shareDecoded.maxIdleSize`. The executor's memory usage log reports the allocator's total, and the part of it charged to the import allocator described below, so that the overhead can be sized for it. diff --git a/docs/source/user-guide/latest/tuning/operators.md b/docs/source/user-guide/latest/tuning/operators.md index 058a6a00cd7..9cdf8a33acd 100644 --- a/docs/source/user-guide/latest/tuning/operators.md +++ b/docs/source/user-guide/latest/tuning/operators.md @@ -63,11 +63,17 @@ Extending reduction to those joins or across small batches is separate follow-up Comet broadcasts the build side of a broadcast join as Arrow IPC batches, which each task that reads the broadcast decodes before building its hash table. By default, the tasks of an executor that read a broadcast at the same time share one decoded copy of it. The first of them decodes the broadcast, and the others wait for that decode and read the -same buffers. The copy is freed when the last of these tasks completes, so a task that starts after that decodes the -broadcast again. Each task still builds its own hash table. - -Set `spark.comet.exec.broadcastExchange.shareDecoded.enabled=false` to have every task decode the whole broadcast for -itself. +same buffers. When the last of these tasks completes, the executor keeps the copy for +`spark.comet.exec.broadcastExchange.shareDecoded.idleTimeout`, 1 second by default, so that a task that starts by then, +such as one of the next wave of tasks of the same stage, reads it instead of decoding the broadcast again. A task that +starts after that decodes the broadcast again. Each task still builds its own hash table. + +The idle copy is JVM-side Arrow memory that neither Spark nor Comet's memory pools account for. The idle copies of all +broadcasts on an executor are capped by `spark.comet.exec.broadcastExchange.shareDecoded.maxIdleSize`, 128 MiB by +default. A copy larger than the cap is freed as soon as no task holds it, and keeping a copy frees the copies that have +been idle longest until the rest fit. Raise the cap to keep larger broadcasts between waves, if the executor's memory +overhead has room for them. Set the timeout to `0` to free the copy as soon as no task holds it. Set `spark.comet.exec.broadcastExchange.shareDecoded.enabled=false` to +have every task decode the whole broadcast for itself. #### Broadcast compression diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 04fb3e537f7..d72154f8a06 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -696,12 +696,42 @@ object CometConf extends ShimCometConf { .doc( "Whether the tasks of an executor that read a Comet broadcast at the same time share " + "one decoded copy of it. The first of them decodes the broadcast, and the others " + - "wait for it instead of decoding it themselves. The copy is freed when the last of " + - "them completes. When disabled, every task decodes the whole broadcast for itself. " + - s"$TUNING_GUIDE.") + "wait for it instead of decoding it themselves. The copy is kept for " + + s"$COMET_EXEC_CONFIG_PREFIX.broadcastExchange.shareDecoded.idleTimeout after the last " + + "of them completes, for the tasks that start by then. When disabled, every task " + + s"decodes the whole broadcast for itself. $TUNING_GUIDE.") .booleanConf .createWithDefault(true) + val COMET_EXEC_BROADCAST_SHARE_DECODED_IDLE_TIMEOUT: ConfigEntry[Long] = + conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcastExchange.shareDecoded.idleTimeout") + .category(CATEGORY_EXEC) + .doc( + "How long an executor keeps the shared decoded copy of a Comet broadcast after the " + + "last task reading it completes, when " + + s"${COMET_EXEC_BROADCAST_SHARE_DECODED_ENABLED.key} is true. A task that starts by " + + "then, such as one of the next wave of tasks of the same stage, reads the copy " + + "instead of decoding the broadcast again. The idle copy is JVM-side Arrow memory " + + "that neither Spark nor Comet's memory pools account for, and " + + s"$COMET_EXEC_CONFIG_PREFIX.broadcastExchange.shareDecoded.maxIdleSize caps it. Set " + + s"to 0 to free the copy as soon as no task holds it. $TUNING_GUIDE.") + .timeConf(TimeUnit.MILLISECONDS) + .checkValue(_ >= 0, "The idle timeout of a shared broadcast decode must not be negative") + .createWithDefault(TimeUnit.SECONDS.toMillis(1)) + + val COMET_EXEC_BROADCAST_SHARE_DECODED_MAX_IDLE_SIZE: ConfigEntry[Long] = + conf(s"$COMET_EXEC_CONFIG_PREFIX.broadcastExchange.shareDecoded.maxIdleSize") + .category(CATEGORY_EXEC) + .doc( + "The most bytes of idle shared decoded broadcast copies that an executor keeps, over " + + "all broadcasts, under " + + s"${COMET_EXEC_BROADCAST_SHARE_DECODED_IDLE_TIMEOUT.key}. A copy larger than this " + + "is freed as soon as no task holds it. Keeping a copy frees the copies that have " + + s"been idle longest until the rest fit. $TUNING_GUIDE.") + .bytesConf(ByteUnit.BYTE) + .checkValue(_ >= 0, "The idle size of shared broadcast decodes must not be negative") + .createWithDefault(128L * 1024 * 1024) + val COMET_BROADCAST_PRODUCER_COALESCE_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.broadcastExchange.producerCoalesce.enabled") .category(CATEGORY_EXEC) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index 175c46a4b14..b7a42796485 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -230,7 +230,12 @@ case class CometBroadcastExchangeExec( override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { val broadcasted = executeBroadcast[CometBroadcastValue]() - new CometBatchRDD(sparkContext, getNumPartitions(), broadcasted, shareDecoded) + new CometBatchRDD( + sparkContext, + getNumPartitions(), + broadcasted, + shareDecoded, + shareDecodedRetention) } // After https://issues.apache.org/jira/browse/SPARK-48195, Spark plan will cache created RDD. @@ -242,12 +247,22 @@ case class CometBroadcastExchangeExec( } val broadcasted = executeBroadcast[CometBroadcastValue]() - new CometBatchRDD(sparkContext, numPartitions, broadcasted, shareDecoded) + new CometBatchRDD( + sparkContext, + numPartitions, + broadcasted, + shareDecoded, + shareDecodedRetention) } private def shareDecoded: Boolean = CometConf.COMET_EXEC_BROADCAST_SHARE_DECODED_ENABLED.get(conf) + private def shareDecodedRetention: SharedDecode.Retention = + SharedDecode.Retention( + CometConf.COMET_EXEC_BROADCAST_SHARE_DECODED_IDLE_TIMEOUT.get(conf), + CometConf.COMET_EXEC_BROADCAST_SHARE_DECODED_MAX_IDLE_SIZE.get(conf)) + override protected[sql] def doExecuteBroadcast[T](): broadcast.Broadcast[T] = { try { relationFuture.get(timeout, TimeUnit.SECONDS).asInstanceOf[broadcast.Broadcast[T]] @@ -310,12 +325,16 @@ object CometBroadcastExchangeExec extends CometSink[BroadcastExchangeExec] { * @param shareDecoded * whether the tasks of an executor that read the broadcast at the same time share one decoded * copy of it, through [[CometBroadcastValue]] + * @param retention + * how long an executor keeps that copy after the last task reading it completes, and the cap on + * the bytes of the idle copies that it keeps */ class CometBatchRDD( sc: SparkContext, val numPartitions: Int, value: broadcast.Broadcast[CometBroadcastValue], - shareDecoded: Boolean) + shareDecoded: Boolean, + retention: SharedDecode.Retention) extends RDD[ColumnarBatch](sc, Nil) { override def getPartitions: Array[Partition] = (0 until numPartitions).toArray.map { i => @@ -326,7 +345,7 @@ class CometBatchRDD( val partition = split.asInstanceOf[CometBatchPartition] val source = this.getClass.getSimpleName if (shareDecoded && context != null) { - partition.value.value.batches(context, source) + partition.value.value.batches(context, source, retention) } else { partition.value.value.chunks.iterator.flatMap(Utils.decodeBatches(_, source)) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastValue.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastValue.scala index 05547425667..6c30c6a767e 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastValue.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastValue.scala @@ -35,6 +35,7 @@ import org.apache.spark.{SparkEnv, TaskContext} import org.apache.spark.io.CompressionCodec import org.apache.spark.sql.comet.util.Utils import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.{ThreadUtils, Utils => SparkUtils} import org.apache.spark.util.io.ChunkedByteBuffer import org.apache.comet.CometArrowAllocator @@ -43,7 +44,7 @@ import org.apache.comet.vector.NativeUtil /** * The value of a Comet broadcast: the Arrow IPC streams that [[Utils.serializeBatches]] wrote, * compressed with Spark's codec. It lets the tasks of an executor that read the broadcast at the - * same time share one decoded copy of it. + * same time, or soon after one another, share one decoded copy of it. * * Without this, every task that reads the broadcast decompresses and decodes all of it. Spark * keeps one deserialized copy of a broadcast's value per executor, so the tasks of an executor @@ -61,8 +62,11 @@ private[comet] final class CometBroadcastValue(val chunks: Array[ChunkedByteBuff @transient private lazy val shared = new SharedDecode(chunks, SharedDecode.decode) /** See [[SharedDecode.batches]]. */ - def batches(context: TaskContext, source: String): Iterator[ColumnarBatch] = - shared.batches(context, source) + def batches( + context: TaskContext, + source: String, + retention: SharedDecode.Retention): Iterator[ColumnarBatch] = + shared.batches(context, source, retention) /** See [[SharedDecode.holders]]. */ def holders: Int = shared.holders @@ -76,59 +80,109 @@ private[comet] final class CometBroadcastValue(val chunks: Array[ChunkedByteBuff * [[org.apache.spark.sql.comet.execution.arrow.ColumnarBatchArrowReader]] does, without affecting * another task. * - * The decode is freed when the last task holding it completes, and a task that reads the value - * after that decodes it again. A native join keeps a build side of one batch, which is what - * coalescing makes of a broadcast, for as long as its task runs, so for such a broadcast the - * decode holds no memory that the tasks would not hold anyway. A native join copies a build side - * of several batches, so the decode keeps one copy of such a broadcast that the tasks would have - * freed once they built their tables. + * When the last task holding the decode completes, the decode is kept for the idle timeout of + * that task's read, so that a task that comes by then, such as one of the next wave of tasks of + * the stage, reads it too. The decode is freed once the timeout passes with no task taking it, or + * at once if the timeout is 0, and a task that reads the value after that decodes it again. The + * idle decodes of all broadcasts on the executor are capped in bytes: a decode larger than the + * cap is freed at once, and keeping a decode frees the decodes that have been idle longest, until + * the idle decodes fit the cap again. A native join keeps a build side of one batch, which is + * what coalescing makes of a broadcast, for as long as its task runs, so for such a broadcast the + * decode holds no memory that running tasks would not hold anyway, apart from while it is idle. A + * native join copies a build side of several batches, so the decode keeps one copy of such a + * broadcast that the tasks would have freed once they built their tables. * * @param chunks * the compressed Arrow IPC streams of the broadcast value * @param decode * decodes the streams; `None` makes each task decode them for itself + * @param schedule + * runs a function on a thread of its own once a delay in milliseconds has passed + * @param idleDecodes + * the idle decodes that count against the cap, shared with the other broadcasts */ private[comet] final class SharedDecode( chunks: Array[ChunkedByteBuffer], - decode: (Array[ChunkedByteBuffer], String) => Option[SharedDecode.Decoded]) { - import SharedDecode.{Decoded, KillCheckIntervalMs} - - // The decode that running tasks hold, if there are any. Guarded by `this`. + decode: (Array[ChunkedByteBuffer], String) => Option[SharedDecode.Decoded], + schedule: (Long, () => Unit) => Unit = SharedDecode.schedule, + idleDecodes: SharedDecode.IdleDecodes = SharedDecode.idleDecodes) { + import SharedDecode.{Decoded, Idle, KillCheckIntervalMs, Retention} + + // The decode that running tasks hold, or that waits out its idle timeout. Guarded by `this`. + // An entry is freed only once it is no longer `current`, so an expiry that finds another entry + // here, or none, frees nothing. private var current: Entry = _ /** * The batches of the broadcast value for the task of `context`. As with * [[Utils.decodeBatches]], each batch is valid until the next call to `hasNext` or `next`, or * until the task completes. A task must read them on its own thread, because that is where its - * completion listener runs and gives back the buffers. + * completion listener runs and gives back the buffers. If the task is the last to hold the + * decode, the decode is kept as `retention` says after the task completes. */ - def batches(context: TaskContext, source: String): Iterator[ColumnarBatch] = - new SharedBatchIterator(context, source) + def batches( + context: TaskContext, + source: String, + retention: Retention): Iterator[ColumnarBatch] = + new SharedBatchIterator(context, source, retention) /** The number of holds on the decode, one for each read by a running task. */ def holders: Int = synchronized(if (current == null) 0 else current.holders) - /** Hands out the entry that tasks hold, creating it if none does. True if it did. */ + /** Hands out the entry, held or idle, creating it if there is none. True if it did. */ private def acquire(): (Entry, Boolean) = synchronized { val loading = current == null if (loading) { current = new Entry + } else if (current.holders == 0) { + idleDecodes.remove(current, current.idlePeriods) } current.holders += 1 (current, loading) } - /** Gives back a task's hold on `entry`, and frees the entry once no task holds it. */ - private def release(entry: Entry): Unit = { - val unheld = synchronized { + /** + * Gives back a task's hold on `entry`. Once no task holds the entry, it is kept as `retention` + * says, or freed at once if it may not be kept. + */ + private def release(entry: Entry, retention: Retention): Unit = { + var free = false + var evicted = Seq.empty[Idle] + synchronized { entry.holders -= 1 - val unheld = entry.holders == 0 - if (unheld) { - forget(entry) + if (entry.holders == 0) { + if (retention.idleTimeoutMs > 0 && entry.bytes <= retention.maxIdleBytes && + (current eq entry)) { + entry.idlePeriods += 1 + val idlePeriod = entry.idlePeriods + evicted = idleDecodes.add( + Idle(entry, idlePeriod, entry.bytes, () => expire(entry, idlePeriod)), + retention.maxIdleBytes) + schedule(retention.idleTimeoutMs, () => expire(entry, idlePeriod)) + } else { + forget(entry) + free = true + } + } + } + if (free) { + entry.close() + } + // Outside this object's lock, because each takes the lock of the decode that it frees. + evicted.foreach(_.expire()) + } + + /** Frees `entry` if no task has taken it since the release that began `idlePeriod`. */ + private def expire(entry: Entry, idlePeriod: Long): Unit = { + val free = synchronized { + idleDecodes.remove(entry, idlePeriod) + val idle = entry.holders == 0 && entry.idlePeriods == idlePeriod && (current eq entry) + if (idle) { + current = null } - unheld + idle } - if (unheld) { + if (free) { entry.close() } } @@ -143,6 +197,10 @@ private[comet] final class SharedDecode( private final class Entry { // One hold for each read of the broadcast by a running task. Guarded by the outer `this`. var holders = 0 + // How many times the last holder has given the entry back. An expiry frees the entry only if + // no task has taken it since the release that scheduled the expiry. Guarded by the outer + // `this`. + var idlePeriods = 0L private val done = new CountDownLatch(1) // Written by the task that loads the entry before `done` opens, and read by others after. @volatile private var decoded: Option[Decoded] = None @@ -171,7 +229,10 @@ private[comet] final class SharedDecode( decoded } - /** Frees the decoded batches. Called once no task holds the entry. */ + /** The bytes of the decoded batches, 0 until the entry loads or if it has none. */ + def bytes: Long = decoded.fold(0L)(_.bytes) + + /** Frees the decoded batches. Called once the entry is no longer handed out. */ def close(): Unit = decoded.foreach(_.close()) } @@ -179,7 +240,10 @@ private[comet] final class SharedDecode( * One task's read of the broadcast: views of the shared batches when the entry has them, or the * task's own decode when it does not. */ - private final class SharedBatchIterator(context: TaskContext, source: String) + private final class SharedBatchIterator( + context: TaskContext, + source: String, + retention: Retention) extends Iterator[ColumnarBatch] { // The task takes its hold on the first call to hasNext, which is also when reading the @@ -213,7 +277,7 @@ private[comet] final class SharedDecode( val (entry, loading) = acquire() context.addTaskCompletionListener[Unit] { _ => try closeCurrent() - finally release(entry) + finally release(entry, retention) } try { batches = (if (loading) entry.load(source) else entry.await(context)) match { @@ -245,6 +309,79 @@ private[comet] object SharedDecode { /** How often a task waiting for another task's decode checks whether it has been killed. */ private val KillCheckIntervalMs = 100L + /** + * How long the last task to hold a decode has it kept, and the cap on the bytes of the idle + * decodes of the executor that its keeping it may leave. + */ + final case class Retention(idleTimeoutMs: Long, maxIdleBytes: Long) + + /** Frees a decode as soon as no task holds it. */ + val NoRetention: Retention = Retention(0, 0) + + /** + * An idle decode, `key` being its entry, kept since the release that began `idlePeriod`. + * `expire` frees it unless a task has taken it since. + */ + final case class Idle(key: AnyRef, idlePeriod: Long, bytes: Long, expire: () => Unit) + + /** + * The idle decodes that count against the cap, oldest first. It takes no other lock, so the + * decodes may call it while they hold their own. + */ + final class IdleDecodes { + private val idle = new java.util.LinkedHashMap[AnyRef, Idle] + private var idleBytes = 0L + + /** The bytes of the idle decodes. */ + def bytes: Long = synchronized(idleBytes) + + /** + * Adds `decode`, and removes and returns the decodes that have been idle longest, until the + * rest fit in `maxBytes`. The caller expires what it returns. + */ + def add(decode: Idle, maxBytes: Long): Seq[Idle] = synchronized { + idle.put(decode.key, decode) + idleBytes += decode.bytes + val evicted = ArrayBuffer.empty[Idle] + val oldestFirst = idle.values.iterator + while (idleBytes > maxBytes && oldestFirst.hasNext) { + val oldest = oldestFirst.next() + if (oldest ne decode) { + oldestFirst.remove() + idleBytes -= oldest.bytes + evicted += oldest + } + } + evicted.toSeq + } + + /** Removes the idle decode of `key`, if it is idle since the release that began `period`. */ + def remove(key: AnyRef, idlePeriod: Long): Unit = synchronized { + val decode = idle.get(key) + if (decode != null && decode.idlePeriod == idlePeriod) { + idle.remove(key) + idleBytes -= decode.bytes + } + } + } + + /** The idle decodes of the executor. */ + private lazy val idleDecodes = new IdleDecodes + + /** The thread that frees the decodes that no task has taken for their idle timeouts. */ + private lazy val expiries = + ThreadUtils.newDaemonSingleThreadScheduledExecutor("comet-broadcast-decode-expiry") + + /** Runs `f` on the thread of [[expiries]] once `delayMs` has passed, logging what it throws. */ + private def schedule(delayMs: Long, f: () => Unit): Unit = { + val _ = expiries.schedule( + new Runnable { + override def run(): Unit = SparkUtils.tryLogNonFatalError(f()) + }, + delayMs, + TimeUnit.MILLISECONDS) + } + /** * Decodes the Arrow IPC streams in `chunks`, which [[Utils.serializeBatches]] wrote through * Spark's codec, into buffers of an allocator of their own. Returns `None`, having kept @@ -276,23 +413,25 @@ private[comet] object SharedDecode { } } if (supported) { - result = Some(new Decoded(allocator, batches.toArray)) + result = Some(new Decoded(allocator, batches.toArray, allocator.getAllocatedMemory)) } result } finally { if (result.isEmpty) { - new Decoded(allocator, batches.toArray).close() + new Decoded(allocator, batches.toArray, 0).close() } } } /** * The batches of a broadcast. Each is kept as an `ArrowRecordBatch`, with its schema, and holds - * references on its buffers, which `allocator` allocated, until it is closed. + * references on its buffers, which `allocator` allocated, until it is closed. `bytes` is what + * the allocator held once it had decoded them. */ final class Decoded private[SharedDecode] ( allocator: BufferAllocator, - batches: Array[(Schema, ArrowRecordBatch)]) { + batches: Array[(Schema, ArrowRecordBatch)], + val bytes: Long) { def numBatches: Int = batches.length diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala index 2116f7506d1..0646567b001 100644 --- a/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala +++ b/spark/src/test/scala/org/apache/spark/sql/CometTestBase.scala @@ -152,6 +152,10 @@ abstract class CometTestBase conf.set(CometConf.COMET_PARQUET_UNSIGNED_SMALL_INT_CHECK.key, "false") conf.set(CometConf.COMET_SCAN_ALLOW_DISABLED_PARQUET_VECTORIZED_READER.key, "true") conf.set(CometConf.COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED.key, "true") + // Free a shared broadcast decode as soon as no task holds it. Suites that compare the Arrow + // allocator's memory before and after a step would otherwise see the idle decode of an + // earlier test's broadcast freed in between. + conf.set(CometConf.COMET_EXEC_BROADCAST_SHARE_DECODED_IDLE_TIMEOUT.key, "0") // Fail loudly if a serde declines an operator without stating why, rather than letting the // generic " is not supported" message mask the missing reason. // See https://github.com/apache/datafusion-comet/issues/5230. diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala index 15b7c706490..d26d214f389 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastValueSuite.scala @@ -24,6 +24,7 @@ import java.util.Properties import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.concurrent.atomic.AtomicInteger +import scala.collection.mutable.ArrayBuffer import scala.concurrent.{Await, Future, Promise} import scala.concurrent.duration._ import scala.util.Try @@ -41,10 +42,11 @@ import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.io.ChunkedByteBuffer -import org.apache.comet.{CometArrowAllocator, CometKryoRegistrator} +import org.apache.comet.{CometArrowAllocator, CometConf, CometKryoRegistrator} import org.apache.comet.vector.CometVector class CometBroadcastValueSuite extends CometTestBase { + import SharedDecode.{NoRetention, Retention} private val Timeout = 30.seconds @@ -54,11 +56,11 @@ class CometBroadcastValueSuite extends CometTestBase { val before = CometArrowAllocator.getAllocatedMemory val tasks = (1 to 4).map(i => newTask(i.toLong)) - assert(readAll(shared.batches(tasks.head, "test")) == rows(1000)) + assert(readAll(shared.batches(tasks.head, "test", NoRetention)) == rows(1000)) val sharedCopy = CometArrowAllocator.getAllocatedMemory - before assert(sharedCopy > 0) tasks.tail.init.foreach { task => - assert(readAll(shared.batches(task, "test")) == rows(1000)) + assert(readAll(shared.batches(task, "test", NoRetention)) == rows(1000)) } assert(decodes.get == 1) assert(shared.holders == 3) @@ -68,7 +70,7 @@ class CometBroadcastValueSuite extends CometTestBase { // A consumer that leaves its batches open, as the JVM columnar-to-row path does, has them // closed when it moves on and when its task completes. - assert(readAll(shared.batches(tasks.last, "test"), close = false) == rows(1000)) + assert(readAll(shared.batches(tasks.last, "test", NoRetention), close = false) == rows(1000)) assert(decodes.get == 1) // The last task to complete frees the decode, and a task that comes later decodes again. @@ -80,19 +82,147 @@ class CometBroadcastValueSuite extends CometTestBase { assert(CometArrowAllocator.getAllocatedMemory == before) val later = newTask(5) - assert(readAll(shared.batches(later, "test")) == rows(1000)) + assert(readAll(shared.batches(later, "test", NoRetention)) == rows(1000)) assert(decodes.get == 2) later.markTaskCompleted(None) assert(CometArrowAllocator.getAllocatedMemory == before) } + test("a task that comes before the idle timeout passes reads the decode of earlier tasks") { + val decodes = new AtomicInteger + // The test runs each expiry itself, in place of a timer. + val expiries = ArrayBuffer.empty[(Long, () => Unit)] + val shared = new SharedDecode( + broadcastValue(rows(1000)), + (chunks, source) => { + decodes.incrementAndGet() + SharedDecode.decode(chunks, source) + }, + (delayMs, expire) => expiries += ((delayMs, expire))) + val before = CometArrowAllocator.getAllocatedMemory + + // The last task to complete keeps the decode for the idle timeout of its read. + val first = newTask(1) + assert(readAll(shared.batches(first, "test", Retention(1000, Long.MaxValue))) == rows(1000)) + first.markTaskCompleted(None) + assert(shared.holders == 0) + assert(expiries.map(_._1) == Seq(1000L)) + assert(CometArrowAllocator.getAllocatedMemory > before) + + // A task that comes in the meantime reads the same decode. The expiry that the first task + // scheduled then frees nothing, while the task runs or after it completes. + val second = newTask(2) + assert(readAll(shared.batches(second, "test", Retention(2000, Long.MaxValue))) == rows(1000)) + assert(decodes.get == 1) + expiries.head._2() + assert(shared.holders == 1) + second.markTaskCompleted(None) + assert(expiries.map(_._1) == Seq(1000L, 2000L)) + expiries.head._2() + assert(CometArrowAllocator.getAllocatedMemory > before) + + // The expiry of the last release frees the decode, and a later task decodes again. With no + // idle timeout, its decode is freed as soon as it completes. + expiries(1)._2() + assert(CometArrowAllocator.getAllocatedMemory == before) + val third = newTask(3) + assert(readAll(shared.batches(third, "test", NoRetention)) == rows(1000)) + assert(decodes.get == 2) + third.markTaskCompleted(None) + assert(expiries.size == 2) + assert(CometArrowAllocator.getAllocatedMemory == before) + } + + test("a task that comes after another completes reads its decode, which the timer then frees") { + val (shared, decodes) = countingDecode(broadcastValue(rows(1000))) + val before = CometArrowAllocator.getAllocatedMemory + // Long enough for the second task to start while the decode is kept, short enough that the + // test waits little for the expiry thread to free it. + val retention = Retention(500, Long.MaxValue) + Seq(1L, 2L).foreach { id => + val task = newTask(id) + assert(readAll(shared.batches(task, "test", retention)) == rows(1000)) + task.markTaskCompleted(None) + } + assert(decodes.get == 1) + eventually(timeout(Timeout), interval(10.millis)) { + assert(CometArrowAllocator.getAllocatedMemory == before) + } + assert(shared.holders == 0) + } + + test("idle decodes over the cap are freed, the longest idle first") { + val decoded = SharedDecode.decode(broadcastValue(rows(1000)), "test").get + val size = decoded.bytes + decoded.close() + // Room for one decode of 1000 rows, but not for two. + val retention = Retention(1000, size + size / 2) + val idle = new SharedDecode.IdleDecodes + val decodes = new AtomicInteger + // The test runs the expiries itself, after the cap has freed some of the decodes. + val expiries = ArrayBuffer.empty[() => Unit] + def sharedDecode(rowCount: Int) = + new SharedDecode( + broadcastValue(rows(rowCount)), + (chunks, source) => { + decodes.incrementAndGet() + SharedDecode.decode(chunks, source) + }, + (_, expire) => expiries += expire, + idle) + val a = sharedDecode(1000) + val b = sharedDecode(1000) + val large = sharedDecode(4000) + val before = CometArrowAllocator.getAllocatedMemory + var taskId = 0L + def readAndComplete(shared: SharedDecode): Unit = { + taskId += 1 + val task = newTask(taskId) + assert(readAll(shared.batches(task, "test", retention)).nonEmpty) + task.markTaskCompleted(None) + } + + readAndComplete(a) + assert(idle.bytes == size) + // Keeping b's decode frees a's, which has been idle longer. + readAndComplete(b) + assert(decodes.get == 2) + assert(idle.bytes == size) + assert(CometArrowAllocator.getAllocatedMemory - before == size) + // a decodes again, and keeping that frees b's decode. + readAndComplete(a) + assert(decodes.get == 3) + assert(idle.bytes == size) + assert(CometArrowAllocator.getAllocatedMemory - before == size) + + // A decode larger than the cap is freed as soon as its task completes. + readAndComplete(large) + assert(decodes.get == 4) + assert(idle.bytes == size) + assert(CometArrowAllocator.getAllocatedMemory - before == size) + + // A task that takes an idle decode takes it out of the cap until it completes. + val task = newTask(taskId + 1) + assert(readAll(a.batches(task, "test", retention)) == rows(1000)) + assert(decodes.get == 4) + assert(idle.bytes == 0) + task.markTaskCompleted(None) + assert(idle.bytes == size) + + // The expiries of the decodes that the cap freed free nothing more. + expiries.foreach(_()) + assert(idle.bytes == 0) + assert(Seq(a, b, large).forall(_.holders == 0)) + assert(CometArrowAllocator.getAllocatedMemory == before) + } + test("a task that stops reading part way frees what it read when it completes") { // Without coalescing, so the broadcast has a batch after the one the task stops at. val value = new CometBroadcastValue((serialize(rows(100)) ++ serialize(rows(200).drop(100))).toArray) val before = CometArrowAllocator.getAllocatedMemory val task = newTask(1) - val first = value.batches(task, "test").next() + val first = value.batches(task, "test", NoRetention).next() assert(first.numRows() == 100) task.markTaskCompleted(None) assert(value.holders == 0) @@ -106,7 +236,7 @@ class CometBroadcastValueSuite extends CometTestBase { val waiter = newTask(2) val loaded = gated.startLoad(loader) - val waited = inThread(readAll(gated.shared.batches(waiter, "test"))) + val waited = inThread(readAll(gated.shared.batches(waiter, "test", NoRetention))) gated.awaitHolders(2) Thread.sleep(200) assert(!waited.isCompleted) @@ -129,7 +259,7 @@ class CometBroadcastValueSuite extends CometTestBase { val waiter = newTask(2) val loaded = gated.startLoad(loader) - val waited = inThread(gated.shared.batches(waiter, "test").hasNext) + val waited = inThread(gated.shared.batches(waiter, "test", NoRetention).hasNext) gated.awaitHolders(2) waiter.markInterrupted("killed by the test") val killed = intercept[TaskKilledException](Await.result(waited, Timeout)) @@ -150,7 +280,7 @@ class CometBroadcastValueSuite extends CometTestBase { val waiter = newTask(2) val loaded = gated.startLoad(loader) - val waited = inThread(readAll(gated.shared.batches(waiter, "test"))) + val waited = inThread(readAll(gated.shared.batches(waiter, "test", NoRetention))) gated.awaitHolders(2) gated.finish.countDown() val failure = intercept[IllegalStateException](Await.result(loaded, Timeout)) @@ -160,7 +290,7 @@ class CometBroadcastValueSuite extends CometTestBase { // The failed entry is no longer handed out, so the next task decodes the broadcast again. val later = newTask(3) - assert(readAll(gated.shared.batches(later, "test")) == rows(1000)) + assert(readAll(gated.shared.batches(later, "test", NoRetention)) == rows(1000)) assert(gated.decodes.get == 2) loader.markTaskCompleted(Some(failure)) @@ -180,7 +310,7 @@ class CometBroadcastValueSuite extends CometTestBase { val (shared, decodes) = countingDecode(value) val tasks = (1 to 2).map(i => newTask(i.toLong)) tasks.foreach { task => - val strings = shared.batches(task, "test").flatMap { batch => + val strings = shared.batches(task, "test", NoRetention).flatMap { batch => (0 until batch.numRows()).map { i => if (batch.column(0).isNullAt(i)) null else batch.column(0).getUTF8String(i).toString } @@ -202,7 +332,7 @@ class CometBroadcastValueSuite extends CometTestBase { val tasks = (1 to 2).map(i => newTask(i.toLong)) // The second task leaves each batch open, so moving on to the next one has to close it. tasks.zip(Seq(true, false)).foreach { case (task, close) => - assert(readAll(shared.batches(task, "test"), close) == rows(1000)) + assert(readAll(shared.batches(task, "test", NoRetention), close) == rows(1000)) } assert(decodes.get == 1) tasks.foreach(_.markTaskCompleted(None)) @@ -215,8 +345,8 @@ class CometBroadcastValueSuite extends CometTestBase { val serialized = Utils.serializeBatches(empty.iterator).map(_._2).toList val value = new CometBroadcastValue(Utils.coalesceBroadcastBatches(serialized.iterator)._1) val task = newTask(1) - assert(value.batches(task, "test").map(_.numRows()).sum == 15) - assert(!new CometBroadcastValue(Array.empty).batches(task, "test").hasNext) + assert(value.batches(task, "test", NoRetention).map(_.numRows()).sum == 15) + assert(!new CometBroadcastValue(Array.empty).batches(task, "test", NoRetention).hasNext) task.markTaskCompleted(None) assert(value.holders == 0) } @@ -225,7 +355,7 @@ class CometBroadcastValueSuite extends CometTestBase { val value = new CometBroadcastValue(broadcastValue(rows(100))) // Decoding first gives the instance state that must not travel with it. val task = newTask(1) - assert(readAll(value.batches(task, "test")) == rows(100)) + assert(readAll(value.batches(task, "test", NoRetention)) == rows(100)) val serializers = Seq( new JavaSerializer(new SparkConf()), new KryoSerializer( @@ -238,7 +368,7 @@ class CometBroadcastValueSuite extends CometTestBase { val copy = instance.deserialize[CometBroadcastValue](instance.serialize(value)) assert(copy.holders == 0) val copyTask = newTask(2) - assert(readAll(copy.batches(copyTask, "test")) == rows(100)) + assert(readAll(copy.batches(copyTask, "test", NoRetention)) == rows(100)) copyTask.markTaskCompleted(None) assert(copy.holders == 0) } @@ -252,7 +382,7 @@ class CometBroadcastValueSuite extends CometTestBase { spark.sparkContext.broadcast(new CometBroadcastValue(broadcastValue(rows(100)))) try { Seq(true, false).foreach { share => - val rdd = new CometBatchRDD(spark.sparkContext, 2, broadcast, share) + val rdd = new CometBatchRDD(spark.sparkContext, 2, broadcast, share, NoRetention) val tasks = (1 to 2).map(i => newTask(i.toLong)) rdd.partitions.zip(tasks).foreach { case (partition, task) => assert(readAll(rdd.compute(partition, task)) == rows(100)) @@ -270,7 +400,9 @@ class CometBroadcastValueSuite extends CometTestBase { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false", SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", - SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1") { + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + // Short, so that the test waits little for the expiry thread to free the decode. + CometConf.COMET_EXEC_BROADCAST_SHARE_DECODED_IDLE_TIMEOUT.key -> "100ms") { withParquetTable((0 until 4000).map(i => (i % 400, s"fact-$i")), "fact") { val dim = (0 until 300).map { i => (i, if (i % 9 == 0) null else s"dim-$i", (i, s"s$i"), Seq(i, i + 1)) @@ -289,9 +421,11 @@ class CometBroadcastValueSuite extends CometTestBase { checkSparkAnswerAndOperator( sql(query), Seq(classOf[CometBroadcastExchangeExec], join)) - assert( - CometArrowAllocator.getAllocatedMemory == before, - "a completed task still holds a broadcast") + eventually(timeout(Timeout), interval(10.millis)) { + assert( + CometArrowAllocator.getAllocatedMemory == before, + "the shared decode outlived its idle timeout") + } } } } @@ -449,7 +583,7 @@ class CometBroadcastValueSuite extends CometTestBase { * Starts `task`'s read of the broadcast on a thread of its own and waits until it decodes. */ def startLoad(task: TaskContextImpl): Future[Seq[(Any, Any)]] = { - val loaded = inThread(readAll(shared.batches(task, "test"))) + val loaded = inThread(readAll(shared.batches(task, "test", NoRetention))) assert(decoding.await(Timeout.toSeconds, TimeUnit.SECONDS)) loaded }