Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/pr_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -666,7 +666,7 @@ jobs:
org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite
org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite
org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite
org.apache.spark.sql.CometSparkInternalFunctionsSuite
- name: "expressions"
value: |
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/pr_build_macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ jobs:
org.apache.spark.sql.comet.execution.arrow.CachedBatchRowIteratorSuite
org.apache.spark.sql.comet.execution.arrow.CometArrowWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometStringWriterSuite
org.apache.spark.sql.comet.execution.arrow.CometZstdCompressionCodecSuite
org.apache.spark.sql.comet.execution.arrow.ArrowZstdCompressionCodecSuite
org.apache.spark.sql.CometSparkInternalFunctionsSuite
- name: "expressions"
value: |
Expand Down
10 changes: 8 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ under the License.
hadoop-azure.version does: MiniDFSCluster cannot start when the minicluster is older than
the hadoop-client-api/runtime Spark supplies. The default matches spark-4.1. -->
<hadoop.version>3.4.2</hadoop.version>
<arrow.version>18.3.0</arrow.version>
<arrow.version>20.0.0</arrow.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<spotless.version>2.43.0</spotless.version>
<!-- semanticdb-scalac is published per Scala patch version, so the 2.13 profiles need a
Expand Down Expand Up @@ -273,12 +273,18 @@ under the License.
<version>${arrow.version}</version>
<!-- Arrow's per-buffer IPC compression codecs. Their backing libraries ship with Spark,
so take them from there rather than bundling a second copy: commons-compress (LZ4
frame) and zstd-jni are both on every Spark version Comet supports. -->
frame), lz4-java and zstd-jni are all on every Spark version Comet supports. Arrow
depends on the at.yawk.lz4 fork of lz4-java, which has the same net.jpountz classes
as the org.lz4 artifact Spark ships, so excluding it avoids duplicate classes. -->
<exclusions>
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-compress</artifactId>
</exclusion>
<exclusion>
<groupId>at.yawk.lz4</groupId>
<artifactId>lz4-java</artifactId>
</exclusion>
<exclusion>
<groupId>com.github.luben</groupId>
<artifactId>zstd-jni</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ import scala.collection.mutable
import scala.jdk.CollectionConverters._
import scala.util.control.NonFatal

import org.apache.arrow.compression.CommonsCompressionFactory
import org.apache.arrow.compression.{CommonsCompressionFactory, ZstdCompressionCodec}
import org.apache.arrow.flatbuf.{RecordBatch => FlatBufRecordBatch}
import org.apache.arrow.memory.{ArrowBuf, BufferAllocator}
import org.apache.arrow.vector.{FieldVector, TypeLayout, ValueVector, VectorLoader, VectorSchemaRoot, VectorUnloader}
Expand Down Expand Up @@ -70,11 +70,9 @@ private[comet] object CachedBatchIpc {
*/
def compressionCodec(codecName: String, zstdLevel: Int): CompressionCodec = codecName match {
case "none" => NoCompressionCodec.INSTANCE
// Comet's subclass of Arrow's codec, which corrects the capacity handed to zstd; see
// CometZstdCompressionCodec. Constructed directly rather than through
// CompressionCodec.Factory, which ignores the level and always builds a codec at zstd's
// default.
case "zstd" => new CometZstdCompressionCodec(zstdLevel)
// Constructed directly rather than through CompressionCodec.Factory, which ignores the level
// and always builds a codec at zstd's default.
case "zstd" => new ZstdCompressionCodec(zstdLevel)
// Arrow's LZ4_FRAME format, written with lz4-java rather than Arrow's own pure-Java codec;
// see Lz4FrameCompressionCodec.
case "lz4" => Lz4FrameCompressionCodec
Expand Down Expand Up @@ -567,13 +565,14 @@ private[comet] object CachedBatchIpc {
/**
* The same record batch with every buffer decompressed, as a new batch the caller owns.
*
* `VectorLoader` would do this itself, but arrow-java 18.3.0 leaks on the failure path:
* `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only releases
* them after the whole field has loaded, so if one buffer of a field fails to decompress, every
* buffer of that field decompressed before it is unreachable and never freed. A string column
* is enough to reach it -- its offsets buffer decompresses, then its data buffer throws -- so a
* single corrupt cached batch leaks off-heap for the life of the executor. Doing the
* decompression here keeps every allocation reachable from this method's own error path.
* `VectorLoader` would do this itself, but arrow-java (through 20.0.0) leaks on the failure
* path: `VectorLoader.loadBuffers` decompresses a field's buffers into a local list and only
* releases them after the whole field has loaded, so if one buffer of a field fails to
* decompress, every buffer of that field decompressed before it is unreachable and never freed.
* A string column is enough to reach it -- its offsets buffer decompresses, then its data
* buffer throws -- so a single corrupt cached batch leaks off-heap for the life of the
* executor. Doing the decompression here keeps every allocation reachable from this method's
* own error path.
*
* Buffers are retained before decompressing rather than after, which is the other half of the
* difference. `decompress` consumes a reference to its input on the paths where it allocates,
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -816,7 +816,7 @@ object Utils extends CometTypeShim with Logging {
if (numValues > 0) {
// An Arrow Java vector has no array offset, so validity bit i belongs to value i even when
// the offsets start past zero, and the bitmap is copied from bit 0.
val validityBytes = BitVectorHelper.getValidityBufferSize(numValues).toLong
val validityBytes = BitVectorHelper.getValidityBufferSizeFromCount(numValues).toLong
narrow.getValidityBuffer.setBytes(0, large.getValidityBuffer, 0, validityBytes)
val offsets = narrow.getOffsetBuffer
(0 to numValues).foreach { i =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,23 @@ import org.apache.arrow.compression.ZstdCompressionCodec
import org.apache.arrow.memory.{AllocationListener, AllocationReservation, ArrowBuf, BufferAllocator, BufferManager, ForeignAllocation, RootAllocator}
import org.apache.arrow.memory.rounding.RoundingPolicy
import org.apache.arrow.vector.compression.CompressionUtil
import org.apache.spark.SparkException

import com.github.luben.zstd.Zstd

/**
* Checks that the zstd codec never writes past the buffer it allocates for its output.
* Checks that Arrow's zstd codec, which Comet's cache compresses with, never writes past the
* buffer it allocates for its output.
*
* The buffers come from [[GuardedAllocator]], which hands the codec exactly the capacity it asked
* for and keeps a sentinel-filled guard region right behind it, so an overrun shows up as a
* changed sentinel instead of as silent corruption of whatever the allocator placed next (#7305).
* The defect is explained on [[CometZstdCompressionCodec]]. The inputs are the ones that provoke
* it: bytes that look structured enough for zstd to try entropy coding but do not compress.
* Before Arrow 20.0.0 (apache/arrow-java#1119), `ZstdCompressionCodec.doCompress` allocated
* `compressBound(n) + 8` bytes and pointed zstd 8 bytes in, past the uncompressed-length prefix,
* but still told it the capacity was `compressBound(n) + 8`. zstd only promises not to write past
* the capacity it is given, and while it entropy-codes the literals of a block that turns out not
* to compress it can fill the destination up to that capacity before falling back to a stored
* block, so it could run up to 8 bytes past the buffer. The inputs are the ones that provoke it:
* bytes that look structured enough for zstd to try entropy coding but do not compress.
*/
class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
class ArrowZstdCompressionCodecSuite extends AnyFunSuite with Matchers {

// zstd-jni unpacks its native library into java.io.tmpdir, which the test JVM points at
// target/tmp. Nothing else creates that directory in a fresh checkout.
Expand Down Expand Up @@ -103,22 +106,9 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
}
}

test("Arrow's zstd codec writes past the output buffer it allocates") {
// Documents the defect this suite guards against (apache/arrow-java#1119, fixed in Arrow
// 20.0.0). Once Comet moves to an Arrow with the fix, CometZstdCompressionCodec and this test
// go together.
test("Arrow's zstd codec stays within the output buffer it allocates") {
withAllocators { (root, guarded) =>
val codec = new ZstdCompressionCodec(1)
Lengths.foreach { len =>
(0 until 50).foreach(seed => roundTrip(codec, root, guarded, protoLike(len, seed)))
}
guarded.overruns should be > 0
}
}

test("Comet's zstd codec stays within the output buffer it allocates") {
withAllocators { (root, guarded) =>
val codec = new CometZstdCompressionCodec(1)
var raw = 0
var packed = 0
def count(storedRaw: Boolean): Unit = if (storedRaw) raw += 1 else packed += 1
Expand All @@ -134,19 +124,6 @@ class CometZstdCompressionCodecSuite extends AnyFunSuite with Matchers {
packed should be > 0
}
}

test("a zstd error releases the output buffer and surfaces as an internal error") {
withAllocators { (root, _) =>
val compressed = root.buffer(16)
// zstd reports an error as the negated error code.
val e = intercept[SparkException] {
CometZstdCompressionCodec.checkWritten(-Zstd.errDstSizeTooSmall(), compressed)
}
e.getMessage should include("zstd failed to compress a cached batch")
e.getMessage should include("INTERNAL_ERROR")
root.getAllocatedMemory shouldBe 0
}
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -368,13 +368,14 @@ class UtilsSuite extends CometTestBase {
}

test("serializeBatches writes empty large-offset vectors with 32-bit offsets") {
// A vector that holds no values need not have an offset buffer, so none is read.
// A vector that holds no values need not have an offset buffer, so none is read. A freshly
// constructed vector has none; since Arrow 20.0.0, setValueCount(0) allocates the initial
// offset, so it is not called here.
val strings = new LargeVarCharVector("s", CometArrowAllocator)
val binaries = new LargeVarBinaryVector("b", CometArrowAllocator)
try {
strings.setValueCount(0)
binaries.setValueCount(0)
assert(strings.getOffsetBuffer.capacity == 0)
assert(binaries.getOffsetBuffer.capacity == 0)
val batch = new ColumnarBatch(Array(cometVector(strings), cometVector(binaries)), 0)
assert(
roundTrip(batch) == Seq(
Expand Down Expand Up @@ -554,7 +555,8 @@ class UtilsSuite extends CometTestBase {
val width = BaseLargeVariableWidthVector.OFFSET_WIDTH.toLong

val validity =
CometArrowAllocator.buffer(BitVectorHelper.getValidityBufferSize(values.length).toLong)
CometArrowAllocator.buffer(
BitVectorHelper.getValidityBufferSizeFromCount(values.length).toLong)
val offsetBuffer = CometArrowAllocator.buffer(offsets.length * width)
val data = CometArrowAllocator.buffer(offsets.last)
val buffers = Seq(validity, offsetBuffer, data)
Expand Down