diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml
index f13101deeeb..f88a1d870d2 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -666,7 +666,7 @@ jobs:
org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite
org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite
- org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite
+ org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite
org.apache.spark.sql.CometSparkInternalFunctionsSuite
- name: "expressions"
value: |
diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml
index ee13480830e..f1b8335a069 100644
--- a/.github/workflows/pr_build_macos.yml
+++ b/.github/workflows/pr_build_macos.yml
@@ -304,7 +304,7 @@ jobs:
org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite
org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite
- org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite
+ org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite
org.apache.spark.sql.CometSparkInternalFunctionsSuite
- name: "expressions"
value: |
diff --git a/pom.xml b/pom.xml
index 6a4b17c002e..23297e3995c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,7 +93,7 @@ under the License.
hadoop-azure.version does: MiniDFSCluster cannot start when the minicluster is older than
the hadoop-client-api/runtime Spark supplies. The default matches spark-4.1. -->
3.4.2
- 18.3.0
+ 20.0.0
1.9.13
2.43.0
+ frame), lz4-java and zstd-jni are all on every Spark version Comet supports. Arrow
+ depends on the at.yawk.lz4 fork of lz4-java, which has the same net.jpountz classes
+ as the org.lz4 artifact Spark ships, so excluding it avoids duplicate classes. -->
org.apache.commons
commons-compress
+
+ at.yawk.lz4
+ lz4-java
+
com.github.luben
zstd-jni
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala
index afc72d04918..53a7e9e5bd5 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala
@@ -26,7 +26,7 @@ import scala.collection.mutable
import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal
-import org.apache.arrow.compression.CommonsCompressionFactory
+import org.apache.arrow.compression.{CommonsCompressionFactory, ZstdCompressionCodec}
import org.apache.arrow.flatbuf.{RecordBatch => FlatBufRecordBatch}
import org.apache.arrow.memory.{ArrowBuf, BufferAllocator}
import org.apache.arrow.vector.{FieldVector, TypeLayout, ValueVector, VectorLoader, VectorSchemaRoot, VectorUnloader}
@@ -70,11 +70,9 @@ private[comet] object CachedBatchIpc {
*/
def compressionCodec(codecName: String, zstdLevel: Int): CompressionCodec = codecName match {
case "none" => NoCompressionCodec.INSTANCE
- // Comet's subclass of Arrow's codec, which corrects the capacity handed to zstd; see
- // CometZstdCompressionCodec. Constructed directly rather than through
- // CompressionCodec.Factory, which ignores the level and always builds a codec at zstd's
- // default.
- case "zstd" => new CometZstdCompressionCodec(zstdLevel)
+ // Constructed directly rather than through CompressionCodec.Factory, which ignores the level
+ // and always builds a codec at zstd's default.
+ case "zstd" => new ZstdCompressionCodec(zstdLevel)
// Arrow's LZ4_FRAME format, written with lz4-java rather than Arrow's own pure-Java codec;
// see Lz4FrameCompressionCodec.
case "lz4" => Lz4FrameCompressionCodec
@@ -567,13 +565,14 @@ private[comet] object CachedBatchIpc {
/**
* The same record batch with every buffer decompressed, as a new batch the caller owns.
*
- * `VectorLoader` would do this itself, but arrow-java 18.3.0 leaks on the failure path:
- * `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only releases
- * them after the whole field has loaded, so if one buffer of a field fails to decompress, every
- * buffer of that field decompressed before it is unreachable and never freed. A string column
- * is enough to reach it -- its offsets buffer decompresses, then its data buffer throws -- so a
- * single corrupt cached batch leaks off-heap for the life of the executor. Doing the
- * decompression here keeps every allocation reachable from this method's own error path.
+ * `VectorLoader` would do this itself, but arrow-java (through 20.0.0) leaks on the failure
+ * path: `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only
+ * releases them after the whole field has loaded, so if one buffer of a field fails to
+ * decompress, every buffer of that field decompressed before it is unreachable and never freed.
+ * A string column is enough to reach it -- its offsets buffer decompresses, then its data
+ * buffer throws -- so a single corrupt cached batch leaks off-heap for the life of the
+ * executor. Doing the decompression here keeps every allocation reachable from this method's
+ * own error path.
*
* Buffers are retained before decompressing rather than after, which is the other half of the
* difference. `decompress` consumes a reference to its input on the paths where it allocates,
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala
deleted file mode 100644
index 6ec35fdd3d0..00000000000
--- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.spark.sql.comet.execution.arrow
-
-import org.apache.arrow.compression.ZstdCompressionCodec
-import org.apache.arrow.memory.{ArrowBuf, BufferAllocator}
-import org.apache.arrow.vector.compression.CompressionUtil
-import org.apache.spark.SparkException
-
-import com.github.luben.zstd.Zstd
-
-/**
- * Arrow's zstd codec, with the destination capacity it hands zstd corrected.
- *
- * Arrow 18.3.0's `ZstdCompressionCodec.doCompress` allocates `compressBound(n) + 8` bytes and
- * points zstd 8 bytes in, past the uncompressed-length prefix, but still tells it the capacity is
- * `compressBound(n) + 8`. What zstd guarantees is that it never writes past the capacity it is
- * given, not that it never writes past `compressBound`: while it entropy-codes the literals of a
- * block that turns out not to compress, it fills the destination up to that capacity before it
- * falls back to a stored block, so the attempt can run up to 8 bytes past the buffer. The
- * allocator's power-of-two rounding absorbs that unless the request lands within 8 bytes below a
- * power of two, and when it does, the overrun lands in whatever follows the buffer without any
- * error. With Native Memory Tracking on, the JVM aborts when the buffer is freed (#7305).
- *
- * Fixed upstream by apache/arrow-java#1119 for Arrow 20.0.0; this is that fix. Only `doCompress`
- * differs, so decompression stays Arrow's.
- */
-private[comet] class CometZstdCompressionCodec(level: Int) extends ZstdCompressionCodec(level) {
-
- override protected def doCompress(
- allocator: BufferAllocator,
- uncompressed: ArrowBuf): ArrowBuf = {
- val capacity = Zstd.compressBound(uncompressed.writerIndex())
- val compressed = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH + capacity)
- val written = Zstd.compressUnsafe(
- compressed.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH,
- capacity,
- uncompressed.memoryAddress(),
- uncompressed.writerIndex(),
- level)
- CometZstdCompressionCodec.checkWritten(written, compressed)
- compressed.writerIndex(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH + written)
- compressed
- }
-}
-
-private[comet] object CometZstdCompressionCodec {
-
- /**
- * Releases `compressed` and throws if `written`, the value zstd returned, is an error code.
- * With `compressBound(n)` as the capacity zstd can only fail here by running out of memory, so
- * this is an internal error.
- */
- def checkWritten(written: Long, compressed: ArrowBuf): Unit = {
- if (Zstd.isError(written)) {
- compressed.close()
- throw SparkException.internalError(
- s"zstd failed to compress a cached batch: ${Zstd.getErrorName(written)}")
- }
- }
-}
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
index af434731a2f..bd72c2f5df4 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala
@@ -816,7 +816,7 @@ object Utils extends CometTypeShim with Logging {
if (numValues > 0) {
// An Arrow Java vector has no array offset, so validity bit i belongs to value i even when
// the offsets start past zero, and the bitmap is copied from bit 0.
- val validityBytes = BitVectorHelper.getValidityBufferSize(numValues).toLong
+ val validityBytes = BitVectorHelper.getValidityBufferSizeFromCount(numValues).toLong
narrow.getValidityBuffer.setBytes(0, large.getValidityBuffer, 0, validityBytes)
val offsets = narrow.getOffsetBuffer
(0 to numValues).foreach { i =>
diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala
similarity index 83%
rename from spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala
rename to spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala
index 361ed654c52..87b81719941 100644
--- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala
@@ -31,20 +31,23 @@ import org.apache.arrow.compression.ZstdCompressionCodec
import org.apache.arrow.memory.{AllocationListener, AllocationReservation, ArrowBuf, BufferAllocator, BufferManager, ForeignAllocation, RootAllocator}
import org.apache.arrow.memory.rounding.RoundingPolicy
import org.apache.arrow.vector.compression.CompressionUtil
-import org.apache.spark.SparkException
-
-import com.github.luben.zstd.Zstd
/**
- * Checks that the zstd codec never writes past the buffer it allocates for its output.
+ * Checks that Arrow's zstd codec, which Comet's cache compresses with, never writes past the
+ * buffer it allocates for its output.
*
* The buffers come from [[GuardedAllocator]], which hands the codec exactly the capacity it asked
* for and keeps a sentinel-filled guard region right behind it, so an overrun shows up as a
* changed sentinel instead of as silent corruption of whatever the allocator placed next (#7305).
- * The defect is explained on [[CometZstdCompressionCodec]]. The inputs are the ones that provoke
- * it: bytes that look structured enough for zstd to try entropy coding but do not compress.
+ * Before Arrow 20.0.0 (apache/arrow-java#1119), `ZstdCompressionCodec.doCompress` allocated
+ * `compressBound(n) + 8` bytes and pointed zstd 8 bytes in, past the uncompressed-length prefix,
+ * but still told it the capacity was `compressBound(n) + 8`. zstd only promises not to write past
+ * the capacity it is given, and while it entropy-codes the literals of a block that turns out not
+ * to compress it can fill the destination up to that capacity before falling back to a stored
+ * block, so it could run up to 8 bytes past the buffer. The inputs are the ones that provoke it:
+ * bytes that look structured enough for zstd to try entropy coding but do not compress.
*/
-class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
+class ArrowZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
// zstd-jni unpacks its native library into java.io.tmpdir, which the test JVM points at
// target/tmp. Nothing else creates that directory in a fresh checkout.
@@ -103,22 +106,9 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
}
}
- test("Arrow's zstd codec writes past the output buffer it allocates") {
- // Documents the defect this suite guards against (apache/arrow-java#1119, fixed in Arrow
- // 20.0.0). Once Comet moves to an Arrow with the fix, CometZstdCompressionCodec and this test
- // go together.
+ test("Arrow's zstd codec stays within the output buffer it allocates") {
withAllocators { (root, guarded) =>
val codec = new ZstdCompressionCodec(1)
- Lengths.foreach { len =>
- (0 until 50).foreach(seed => roundTrip(codec, root, guarded, protoLike(len, seed)))
- }
- guarded.overruns should be > 0
- }
- }
-
- test("Comet's zstd codec stays within the output buffer it allocates") {
- withAllocators { (root, guarded) =>
- val codec = new CometZstdCompressionCodec(1)
var raw = 0
var packed = 0
def count(storedRaw: Boolean): Unit = if (storedRaw) raw += 1 else packed += 1
@@ -134,19 +124,6 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
packed should be > 0
}
}
-
- test("a zstd error releases the output buffer and surfaces as an internal error") {
- withAllocators { (root, _) =>
- val compressed = root.buffer(16)
- // zstd reports an error as the negated error code.
- val e = intercept[SparkException] {
- CometZstdCompressionCodec.checkWritten(-Zstd.errDstSizeTooSmall(), compressed)
- }
- e.getMessage should include("zstd failed to compress a cached batch")
- e.getMessage should include("INTERNAL_ERROR")
- root.getAllocatedMemory shouldBe 0
- }
- }
}
/**
diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
index 86a4fe7a899..e50d71b7af3 100644
--- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala
@@ -368,13 +368,14 @@ class UtilsSuite extends CometTestBase {
}
test("serializeBatches writes empty large-offset vectors with 32-bit offsets") {
- // A vector that holds no values need not have an offset buffer, so none is read.
+ // A vector that holds no values need not have an offset buffer, so none is read. A freshly
+ // constructed vector has none; since Arrow 20.0.0, setValueCount(0) allocates the initial
+ // offset, so it is not called here.
val strings = new LargeVarCharVector("s", CometArrowAllocator)
val binaries = new LargeVarBinaryVector("b", CometArrowAllocator)
try {
- strings.setValueCount(0)
- binaries.setValueCount(0)
assert(strings.getOffsetBuffer.capacity == 0)
+ assert(binaries.getOffsetBuffer.capacity == 0)
val batch = new ColumnarBatch(Array(cometVector(strings), cometVector(binaries)), 0)
assert(
roundTrip(batch) == Seq(
@@ -554,7 +555,8 @@ class UtilsSuite extends CometTestBase {
val width = BaseLargeVariableWidthVector.OFFSET_WIDTH.toLong
val validity =
- CometArrowAllocator.buffer(BitVectorHelper.getValidityBufferSize(values.length).toLong)
+ CometArrowAllocator.buffer(
+ BitVectorHelper.getValidityBufferSizeFromCount(values.length).toLong)
val offsetBuffer = CometArrowAllocator.buffer(offsets.length * width)
val data = CometArrowAllocator.buffer(offsets.last)
val buffers = Seq(validity, offsetBuffer, data)