diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala index a062f868be1..e106fd5c67f 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometBroadcastExchangeExec.scala @@ -143,7 +143,7 @@ case class CometBroadcastExchangeExec( val beforeBuild = System.nanoTime() longMetric("collectTime") += NANOSECONDS.toMillis(beforeBuild - beforeCollect) - // Coalesce the many small per-shuffle-block buffers into a single buffer. + // Coalesce the many small per-shuffle-block buffers, usually into a single buffer. // Without this, each consumer task deserializes one Arrow IPC stream per // shuffle block (one per writer task per partition), which is very expensive // when there are hundreds of writer tasks and partitions. See the scaladoc diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index 2fb1dfb5569..a8cbd790375 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -34,7 +34,7 @@ import org.apache.arrow.vector.dictionary.DictionaryProvider.MapDictionaryProvid import org.apache.arrow.vector.ipc.{ArrowStreamReader, ArrowStreamWriter} import org.apache.arrow.vector.types._ import org.apache.arrow.vector.types.pojo.{ArrowType, Field, FieldType, Schema} -import org.apache.arrow.vector.util.VectorSchemaRootAppender +import org.apache.arrow.vector.util.{OversizedAllocationException, VectorSchemaRootAppender} import org.apache.spark.{SparkEnv, SparkException} import org.apache.spark.internal.Logging import org.apache.spark.io.CompressionCodec @@ -360,7 +360,15 @@ object Utils extends CometTypeShim with Logging { } /** - * Coalesces many small Arrow IPC batches into a single batch for broadcasting. + * The most bytes of values a string or binary vector of a coalesced broadcast batch holds. + * `VectorAppender` grows a data buffer by doubling it, and Arrow refuses a buffer of 2^31 + * bytes, so 2^30 is as far as an appended vector can grow (#7192). + */ + private val MaxCoalescedValueBytes = 1L << 30 + + /** + * Coalesces many small Arrow IPC batches into as few batches as Arrow can hold for + * broadcasting, usually one. * * Why this is necessary: The broadcast exchange collects shuffle output by calling * getByteArrayRdd, which serializes each ColumnarBatch independently into its own @@ -375,9 +383,16 @@ object Utils extends CometTypeShim with Logging { * allocation) for each one. With coalescing, we decode and append all batches into one * VectorSchemaRoot on the driver, then re-serialize once. Each consumer task then deserializes * a single Arrow IPC stream. + * + * A broadcast whose string or binary values do not fit one vector is written as several + * batches: a batch that would take a vector past `maxValueBytes` starts the next one. + * + * @param maxValueBytes + * the most bytes of values a string or binary vector of one coalesced batch may hold */ def coalesceBroadcastBatches( - input: Iterator[ChunkedByteBuffer]): (Array[ChunkedByteBuffer], Long, Long) = { + input: Iterator[ChunkedByteBuffer], + maxValueBytes: Long = MaxCoalescedValueBytes): (Array[ChunkedByteBuffer], Long, Long) = { val buffers = input.filterNot(_.size == 0).toArray if (buffers.isEmpty) { return (Array.empty, 0L, 0L) @@ -386,11 +401,45 @@ object Utils extends CometTypeShim with Logging { val allocator = org.apache.comet.CometArrowAllocator .newChildAllocator("broadcast-coalesce", 0, Long.MaxValue) try { + var schema: Schema = null var targetRoot: VectorSchemaRoot = null + // The rows appended to targetRoot, which VSRAppender does not count without columns. + var targetRows = 0L var totalRows = 0L var batchCount = 0 + val coalesced = scala.collection.mutable.ArrayBuffer.empty[ChunkedByteBuffer] val codec = CompressionCodec.createCodec(SparkEnv.get.conf) + + // Writes targetRoot as one Arrow IPC stream, and releases it. + def writeTarget(): Unit = { + if (targetRoot.getSchema.getFields.isEmpty) { + // VSRAppender does not update rowCount with no columns + targetRoot.setRowCount(targetRows.toInt) + } + + assert( + targetRoot.getRowCount.toLong == targetRows, + s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $targetRows") + + val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) + val compressedOutputStream = + new DataOutputStream(codec.compressedOutputStream(outputStream)) + val writer = + new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream)) + try { + writer.start() + writer.writeBatch() + } finally { + writer.close() + } + + coalesced += outputStream.toChunkedByteBuffer + targetRoot.close() + targetRoot = null + targetRows = 0L + } + try { for (bytes <- buffers) { val compressedInputStream = @@ -415,15 +464,23 @@ object Utils extends CometTypeShim with Logging { } while (reader.loadNextBatch()) { val sourceRoot = reader.getVectorSchemaRoot + if (targetRows > 0 && exceedsValueBytes(targetRoot, sourceRoot, maxValueBytes)) { + writeTarget() + } if (targetRoot == null) { - targetRoot = VectorSchemaRoot.create(sourceRoot.getSchema, allocator) + // Every coalesced batch takes the first one's schema, so a batch that does not + // match it still leaves the broadcast uncoalesced. + if (schema == null) schema = sourceRoot.getSchema + targetRoot = VectorSchemaRoot.create(schema, allocator) targetRoot.allocateNew() } try { // Appending needs offsets that start at zero, which serializeBatches writes. VectorSchemaRootAppender.append(targetRoot, sourceRoot) } catch { - case e: IllegalArgumentException => + // Arrow throws OversizedAllocationException for a batch that is too large to + // append even after the split above. + case e @ (_: IllegalArgumentException | _: OversizedAllocationException) => logWarning( "Arrow batches cannot be appended during BroadcastExchange coalescing; " + "skipping coalesce", @@ -432,6 +489,7 @@ object Utils extends CometTypeShim with Logging { targetRoot = null return (buffers, 0L, 0L) } + targetRows += sourceRoot.getRowCount totalRows += sourceRoot.getRowCount batchCount += 1 } @@ -443,31 +501,12 @@ object Utils extends CometTypeShim with Logging { if (targetRoot == null) { return (Array.empty, 0L, 0L) } + writeTarget() - if (targetRoot.getSchema.getFields.isEmpty) { - // VSRAppender does not update rowCount with no columns - targetRoot.setRowCount(totalRows.toInt) - } - - assert( - targetRoot.getRowCount.toLong == totalRows, - s"Row count mismatch after coalesce: ${targetRoot.getRowCount} != $totalRows") - - logInfo(s"Coalesced $batchCount broadcast batches into 1 ($totalRows rows)") - - val outputStream = new ChunkedByteBufferOutputStream(1024 * 1024, ByteBuffer.allocate) - val compressedOutputStream = - new DataOutputStream(codec.compressedOutputStream(outputStream)) - val writer = - new ArrowStreamWriter(targetRoot, null, Channels.newChannel(compressedOutputStream)) - try { - writer.start() - writer.writeBatch() - } finally { - writer.close() - } + logInfo( + s"Coalesced $batchCount broadcast batches into ${coalesced.length} ($totalRows rows)") - (Array(outputStream.toChunkedByteBuffer), batchCount.toLong, totalRows) + (coalesced.toArray, batchCount.toLong, totalRows) } finally { if (targetRoot != null) { targetRoot.close() @@ -478,6 +517,30 @@ object Utils extends CometTypeShim with Logging { } } + /** + * Whether appending `source` to `target` would take a string or binary vector of `target`, at + * any depth, past `maxValueBytes` of values. + */ + private def exceedsValueBytes( + target: VectorSchemaRoot, + source: VectorSchemaRoot, + maxValueBytes: Long): Boolean = { + def exceeds(t: FieldVector, s: FieldVector): Boolean = { + val valueBytes = (t, s) match { + case (tv: BaseVariableWidthVector, sv: BaseVariableWidthVector) => + tv.sizeOfValueBuffer.toLong + sv.sizeOfValueBuffer + case _ => 0L + } + valueBytes > maxValueBytes || + t.getChildrenFromFields.asScala.zip(s.getChildrenFromFields.asScala).exists { + case (tc, sc) => exceeds(tc, sc) + } + } + target.getFieldVectors.asScala.zip(source.getFieldVectors.asScala).exists { case (t, s) => + exceeds(t, s) + } + } + /** * Whether every column in `batch` is an Arrow-backed `CometVector`, so [[getBatchFieldVectors]] * can hand out its vectors directly. Callers that may receive batches from a plan they did not diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala index 483309b0a90..29c4547ccd3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala @@ -27,6 +27,7 @@ import scala.jdk.CollectionConverters._ import org.apache.arrow.c.CDataDictionaryProvider import org.apache.arrow.memory.BufferAllocator import org.apache.arrow.vector.{BaseLargeVariableWidthVector, BitVectorHelper, IntVector, LargeVarBinaryVector, LargeVarCharVector, ValueVector, VarBinaryVector, VarCharVector, VariableWidthFieldVector} +import org.apache.arrow.vector.complex.StructVector import org.apache.arrow.vector.dictionary.{Dictionary, DictionaryProvider} import org.apache.arrow.vector.ipc.message.ArrowFieldNode import org.apache.arrow.vector.types.pojo.{ArrowType, DictionaryEncoding, Field, FieldType} @@ -343,6 +344,47 @@ class UtilsSuite extends CometTestBase { } } + test("coalesceBroadcastBatches starts a new batch before a string vector passes the cap") { + // VectorAppender cannot grow a string or binary vector past 1 GiB of values (#7192), so a + // batch that would take one past the cap starts the next coalesced batch. A small cap shows + // it, and only the strings nested in the struct reach it. + val numBatches = 4 + val vectors = (0 until numBatches).map { b => + ( + fill(new VarCharVector("s", CometArrowAllocator), Seq(s"$b", null)), + structOfStrings("st", Seq(s"abc$b", s"def$b"))) + } + try { + val batches = vectors.map { case (s, st) => + new ColumnarBatch(Array(cometVector(s), cometVector(st)), 2) + } + val bufs = Utils.serializeBatches(batches.iterator).map(_._2).toSeq.iterator + // Each batch holds 8 bytes of nested values, so two fit under the cap and a third does not. + val (coalesced, batchCount, totalRows) = + Utils.coalesceBroadcastBatches(bufs, maxValueBytes = 20L) + assert(batchCount == numBatches) + assert(totalRows == 2L * numBatches) + + val decoded = coalesced.toSeq.map { buf => + val it = Utils.decodeBatches(buf, "test") + val out = it.next() + val values = ( + stringValues(out.column(0), out.numRows()), + stringValues(out.column(1).getChild(0), out.numRows())) + assert(!it.hasNext) + values + } + assert(decoded.map(_._1.length) == Seq(4, 4)) + assert(decoded.flatMap(_._1) == (0 until numBatches).flatMap(b => Seq(s"$b", null))) + assert(decoded.flatMap(_._2) == (0 until numBatches).flatMap(b => Seq(s"abc$b", s"def$b"))) + } finally { + vectors.foreach { case (s, st) => + s.close() + st.close() + } + } + } + test("serializeBatches releases the vectors it made when a later column fails") { // A materialized constant and a 32-bit copy of a large-offset column are both made before the // last column is refused. Nothing outside serializeBatches ever holds them, so it has to @@ -418,6 +460,18 @@ class UtilsSuite extends CometTestBase { vector } + /** A struct column `name` whose one child, `c`, holds `values`. */ + private def structOfStrings(name: String, values: Seq[String]): StructVector = { + val struct = StructVector.empty(name, CometArrowAllocator) + val child = + struct.addOrGet("c", FieldType.nullable(ArrowType.Utf8.INSTANCE), classOf[VarCharVector]) + struct.allocateNew() + fill(child, values) + values.indices.foreach(struct.setIndexDefined) + struct.setValueCount(values.length) + struct + } + /** * Loads `values` into `vector` the way an imported slice holds them: the data buffer starts * with the bytes of `skipped`, which no value covers, so the offsets start past zero.