From 12633a72120aa1a79453af68d6e4e2e8dfdcbe0c Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 17 Nov 2026 06:52:07 -0700 Subject: [PATCH] build: upgrade arrow-java to 20.0.0 Arrow 20.0.0 ships the zstd capacity fix (apache/arrow-java#1119) that CometZstdCompressionCodec backported, so the subclass goes and the cache uses Arrow's ZstdCompressionCodec. Its guarded-allocator test now runs against Arrow's codec. arrow-compression now depends on the at.yawk.lz4 fork of lz4-java, which has the same net.jpountz classes as the org.lz4 jar Spark ships, so it is excluded like the other codec libraries. BitVectorHelper.getValidityBufferSize was removed in favour of getValidityBufferSizeFromCount. setValueCount(0) on a large-offset vector now allocates the initial offset, so the UtilsSuite test that covers a vector with no offset buffer uses a fresh vector instead. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/pr_build_linux.yml | 2 +- .github/workflows/pr_build_macos.yml | 2 +- pom.xml | 10 ++- .../execution/arrow/CachedBatchIpc.scala | 25 +++--- .../arrow/CometZstdCompressionCodec.scala | 78 ------------------- .../apache/spark/sql/comet/util/Utils.scala | 2 +- ...a => ArrowZstdCompressionCodecSuite.scala} | 45 +++-------- .../spark/sql/comet/util/UtilsSuite.scala | 10 ++- 8 files changed, 40 insertions(+), 134 deletions(-) delete mode 100644 spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala rename spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/{CometZstdCompressionCodecSuite.scala => ArrowZstdCompressionCodecSuite.scala} (83%) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index f13101deeeb..f88a1d870d2 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -666,7 +666,7 @@ jobs: org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite - org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite + org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite org.apache.spark.sql.CometSparkInternalFunctionsSuite - name: "expressions" value: | diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index ee13480830e..f1b8335a069 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -304,7 +304,7 @@ jobs: org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite - org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite + org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite org.apache.spark.sql.CometSparkInternalFunctionsSuite - name: "expressions" value: | diff --git a/pom.xml b/pom.xml index 6a4b17c002e..23297e3995c 100644 --- a/pom.xml +++ b/pom.xml @@ -93,7 +93,7 @@ under the License. hadoop-azure.version does: MiniDFSCluster cannot start when the minicluster is older than the hadoop-client-api/runtime Spark supplies. The default matches spark-4.1. --> 3.4.2 - 18.3.0 + 20.0.0 1.9.13 2.43.0 + frame), lz4-java and zstd-jni are all on every Spark version Comet supports. Arrow + depends on the at.yawk.lz4 fork of lz4-java, which has the same net.jpountz classes + as the org.lz4 artifact Spark ships, so excluding it avoids duplicate classes. --> org.apache.commons commons-compress + + at.yawk.lz4 + lz4-java + com.github.luben zstd-jni diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala index afc72d04918..53a7e9e5bd5 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CachedBatchIpc.scala @@ -26,7 +26,7 @@ import scala.collection.mutable import scala.jdk.CollectionConverters._ import scala.util.control.NonFatal -import org.apache.arrow.compression.CommonsCompressionFactory +import org.apache.arrow.compression.{CommonsCompressionFactory, ZstdCompressionCodec} import org.apache.arrow.flatbuf.{RecordBatch => FlatBufRecordBatch} import org.apache.arrow.memory.{ArrowBuf, BufferAllocator} import org.apache.arrow.vector.{FieldVector, TypeLayout, ValueVector, VectorLoader, VectorSchemaRoot, VectorUnloader} @@ -70,11 +70,9 @@ private[comet] object CachedBatchIpc { */ def compressionCodec(codecName: String, zstdLevel: Int): CompressionCodec = codecName match { case "none" => NoCompressionCodec.INSTANCE - // Comet's subclass of Arrow's codec, which corrects the capacity handed to zstd; see - // CometZstdCompressionCodec. Constructed directly rather than through - // CompressionCodec.Factory, which ignores the level and always builds a codec at zstd's - // default. - case "zstd" => new CometZstdCompressionCodec(zstdLevel) + // Constructed directly rather than through CompressionCodec.Factory, which ignores the level + // and always builds a codec at zstd's default. + case "zstd" => new ZstdCompressionCodec(zstdLevel) // Arrow's LZ4_FRAME format, written with lz4-java rather than Arrow's own pure-Java codec; // see Lz4FrameCompressionCodec. case "lz4" => Lz4FrameCompressionCodec @@ -567,13 +565,14 @@ private[comet] object CachedBatchIpc { /** * The same record batch with every buffer decompressed, as a new batch the caller owns. * - * `VectorLoader` would do this itself, but arrow-java 18.3.0 leaks on the failure path: - * `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only releases - * them after the whole field has loaded, so if one buffer of a field fails to decompress, every - * buffer of that field decompressed before it is unreachable and never freed. A string column - * is enough to reach it -- its offsets buffer decompresses, then its data buffer throws -- so a - * single corrupt cached batch leaks off-heap for the life of the executor. Doing the - * decompression here keeps every allocation reachable from this method's own error path. + * `VectorLoader` would do this itself, but arrow-java (through 20.0.0) leaks on the failure + * path: `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only + * releases them after the whole field has loaded, so if one buffer of a field fails to + * decompress, every buffer of that field decompressed before it is unreachable and never freed. + * A string column is enough to reach it -- its offsets buffer decompresses, then its data + * buffer throws -- so a single corrupt cached batch leaks off-heap for the life of the + * executor. Doing the decompression here keeps every allocation reachable from this method's + * own error path. * * Buffers are retained before decompressing rather than after, which is the other half of the * difference. `decompress` consumes a reference to its input on the paths where it allocates, diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala deleted file mode 100644 index 6ec35fdd3d0..00000000000 --- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodec.scala +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.spark.sql.comet.execution.arrow - -import org.apache.arrow.compression.ZstdCompressionCodec -import org.apache.arrow.memory.{ArrowBuf, BufferAllocator} -import org.apache.arrow.vector.compression.CompressionUtil -import org.apache.spark.SparkException - -import com.github.luben.zstd.Zstd - -/** - * Arrow's zstd codec, with the destination capacity it hands zstd corrected. - * - * Arrow 18.3.0's `ZstdCompressionCodec.doCompress` allocates `compressBound(n) + 8` bytes and - * points zstd 8 bytes in, past the uncompressed-length prefix, but still tells it the capacity is - * `compressBound(n) + 8`. What zstd guarantees is that it never writes past the capacity it is - * given, not that it never writes past `compressBound`: while it entropy-codes the literals of a - * block that turns out not to compress, it fills the destination up to that capacity before it - * falls back to a stored block, so the attempt can run up to 8 bytes past the buffer. The - * allocator's power-of-two rounding absorbs that unless the request lands within 8 bytes below a - * power of two, and when it does, the overrun lands in whatever follows the buffer without any - * error. With Native Memory Tracking on, the JVM aborts when the buffer is freed (#7305). - * - * Fixed upstream by apache/arrow-java#1119 for Arrow 20.0.0; this is that fix. Only `doCompress` - * differs, so decompression stays Arrow's. - */ -private[comet] class CometZstdCompressionCodec(level: Int) extends ZstdCompressionCodec(level) { - - override protected def doCompress( - allocator: BufferAllocator, - uncompressed: ArrowBuf): ArrowBuf = { - val capacity = Zstd.compressBound(uncompressed.writerIndex()) - val compressed = allocator.buffer(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH + capacity) - val written = Zstd.compressUnsafe( - compressed.memoryAddress() + CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH, - capacity, - uncompressed.memoryAddress(), - uncompressed.writerIndex(), - level) - CometZstdCompressionCodec.checkWritten(written, compressed) - compressed.writerIndex(CompressionUtil.SIZE_OF_UNCOMPRESSED_LENGTH + written) - compressed - } -} - -private[comet] object CometZstdCompressionCodec { - - /** - * Releases `compressed` and throws if `written`, the value zstd returned, is an error code. - * With `compressBound(n)` as the capacity zstd can only fail here by running out of memory, so - * this is an internal error. - */ - def checkWritten(written: Long, compressed: ArrowBuf): Unit = { - if (Zstd.isError(written)) { - compressed.close() - throw SparkException.internalError( - s"zstd failed to compress a cached batch: ${Zstd.getErrorName(written)}") - } - } -} diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala index af434731a2f..bd72c2f5df4 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala @@ -816,7 +816,7 @@ object Utils extends CometTypeShim with Logging { if (numValues > 0) { // An Arrow Java vector has no array offset, so validity bit i belongs to value i even when // the offsets start past zero, and the bitmap is copied from bit 0. - val validityBytes = BitVectorHelper.getValidityBufferSize(numValues).toLong + val validityBytes = BitVectorHelper.getValidityBufferSizeFromCount(numValues).toLong narrow.getValidityBuffer.setBytes(0, large.getValidityBuffer, 0, validityBytes) val offsets = narrow.getOffsetBuffer (0 to numValues).foreach { i => diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala similarity index 83% rename from spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala rename to spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala index 361ed654c52..87b81719941 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/CometZstdCompressionCodecSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/execution/arrow/ArrowZstdCompressionCodecSuite.scala @@ -31,20 +31,23 @@ import org.apache.arrow.compression.ZstdCompressionCodec import org.apache.arrow.memory.{AllocationListener, AllocationReservation, ArrowBuf, BufferAllocator, BufferManager, ForeignAllocation, RootAllocator} import org.apache.arrow.memory.rounding.RoundingPolicy import org.apache.arrow.vector.compression.CompressionUtil -import org.apache.spark.SparkException - -import com.github.luben.zstd.Zstd /** - * Checks that the zstd codec never writes past the buffer it allocates for its output. + * Checks that Arrow's zstd codec, which Comet's cache compresses with, never writes past the + * buffer it allocates for its output. * * The buffers come from [[GuardedAllocator]], which hands the codec exactly the capacity it asked * for and keeps a sentinel-filled guard region right behind it, so an overrun shows up as a * changed sentinel instead of as silent corruption of whatever the allocator placed next (#7305). - * The defect is explained on [[CometZstdCompressionCodec]]. The inputs are the ones that provoke - * it: bytes that look structured enough for zstd to try entropy coding but do not compress. + * Before Arrow 20.0.0 (apache/arrow-java#1119), `ZstdCompressionCodec.doCompress` allocated + * `compressBound(n) + 8` bytes and pointed zstd 8 bytes in, past the uncompressed-length prefix, + * but still told it the capacity was `compressBound(n) + 8`. zstd only promises not to write past + * the capacity it is given, and while it entropy-codes the literals of a block that turns out not + * to compress it can fill the destination up to that capacity before falling back to a stored + * block, so it could run up to 8 bytes past the buffer. The inputs are the ones that provoke it: + * bytes that look structured enough for zstd to try entropy coding but do not compress. */ -class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers { +class ArrowZstdCompressionCodecSuite extends AnyFunSuite with Matchers { // zstd-jni unpacks its native library into java.io.tmpdir, which the test JVM points at // target/tmp. Nothing else creates that directory in a fresh checkout. @@ -103,22 +106,9 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers { } } - test("Arrow's zstd codec writes past the output buffer it allocates") { - // Documents the defect this suite guards against (apache/arrow-java#1119, fixed in Arrow - // 20.0.0). Once Comet moves to an Arrow with the fix, CometZstdCompressionCodec and this test - // go together. + test("Arrow's zstd codec stays within the output buffer it allocates") { withAllocators { (root, guarded) => val codec = new ZstdCompressionCodec(1) - Lengths.foreach { len => - (0 until 50).foreach(seed => roundTrip(codec, root, guarded, protoLike(len, seed))) - } - guarded.overruns should be > 0 - } - } - - test("Comet's zstd codec stays within the output buffer it allocates") { - withAllocators { (root, guarded) => - val codec = new CometZstdCompressionCodec(1) var raw = 0 var packed = 0 def count(storedRaw: Boolean): Unit = if (storedRaw) raw += 1 else packed += 1 @@ -134,19 +124,6 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers { packed should be > 0 } } - - test("a zstd error releases the output buffer and surfaces as an internal error") { - withAllocators { (root, _) => - val compressed = root.buffer(16) - // zstd reports an error as the negated error code. - val e = intercept[SparkException] { - CometZstdCompressionCodec.checkWritten(-Zstd.errDstSizeTooSmall(), compressed) - } - e.getMessage should include("zstd failed to compress a cached batch") - e.getMessage should include("INTERNAL_ERROR") - root.getAllocatedMemory shouldBe 0 - } - } } /** diff --git a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala index 86a4fe7a899..e50d71b7af3 100644 --- a/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala +++ b/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala @@ -368,13 +368,14 @@ class UtilsSuite extends CometTestBase { } test("serializeBatches writes empty large-offset vectors with 32-bit offsets") { - // A vector that holds no values need not have an offset buffer, so none is read. + // A vector that holds no values need not have an offset buffer, so none is read. A freshly + // constructed vector has none; since Arrow 20.0.0, setValueCount(0) allocates the initial + // offset, so it is not called here. val strings = new LargeVarCharVector("s", CometArrowAllocator) val binaries = new LargeVarBinaryVector("b", CometArrowAllocator) try { - strings.setValueCount(0) - binaries.setValueCount(0) assert(strings.getOffsetBuffer.capacity == 0) + assert(binaries.getOffsetBuffer.capacity == 0) val batch = new ColumnarBatch(Array(cometVector(strings), cometVector(binaries)), 0) assert( roundTrip(batch) == Seq( @@ -554,7 +555,8 @@ class UtilsSuite extends CometTestBase { val width = BaseLargeVariableWidthVector.OFFSET_WIDTH.toLong val validity = - CometArrowAllocator.buffer(BitVectorHelper.getValidityBufferSize(values.length).toLong) + CometArrowAllocator.buffer( + BitVectorHelper.getValidityBufferSizeFromCount(values.length).toLong) val offsetBuffer = CometArrowAllocator.buffer(offsets.length * width) val data = CometArrowAllocator.buffer(offsets.last) val buffers = Seq(validity, offsetBuffer, data)