From 2669ab84cc6354f5f2d5d50920dc52610364fd5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 12 Nov 2026 15:16:57 +0100 Subject: [PATCH] Add an option to fully validate IPC batches when reading them IPC readers trust the offsets, views and run ends of the batches they load. A malformed batch, such as one with an offset past the end of its data buffer, is read past the end of the buffer, without bounds checks when arrow.enable_unsafe_memory_access is set. Add ArrowReader.setValidateBatches, off by default, which checks every record batch and dictionary batch with ValueVectorUtility.validateFull once it is loaded, clears it if the check fails and throws the validation error. A delta dictionary is checked before it is appended. For this, validateFull now works on every vector type an IPC batch can hold: - ListViewVector and LargeListViewVector are supported by the type, buffer and data validators. They threw UnsupportedOperationException. - RunEndEncodedVector is supported by the data validator: run ends must be strictly increasing, and each run must have a value. - FixedSizeListVector no longer reads an offset buffer it does not have, which threw UnsupportedOperationException for any non-empty vector. - MapVector is accepted by the type validator, which rejected every map as not being of type List. - Views are checked against the data buffers before their values are read. - Dense union offsets must not be negative, and type ids must have a sub-vector. TestRoundTrip now reads every batch with the option on. The test populator for LargeListViewVector wrote 4-byte sizes at the wrong stride and now writes 64-bit offsets and sizes. --- .../apache/arrow/vector/ipc/ArrowReader.java | 49 +++++ .../validate/ValidateVectorBufferVisitor.java | 39 ++++ .../validate/ValidateVectorDataVisitor.java | 138 +++++++++++++- .../validate/ValidateVectorTypeVisitor.java | 27 ++- .../vector/TestVariableWidthViewVector.java | 5 +- .../vector/ipc/TestArrowReaderValidation.java | 174 ++++++++++++++++++ .../arrow/vector/ipc/TestRoundTrip.java | 3 + .../testing/ValueVectorDataPopulator.java | 4 +- .../validate/TestValidateVectorFull.java | 111 +++++++++++ 9 files changed, 543 insertions(+), 7 deletions(-) create mode 100644 vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderValidation.java diff --git a/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java b/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java index 7f4addf2d0..ba839aae1d 100644 --- a/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java +++ b/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java @@ -35,6 +35,7 @@ import org.apache.arrow.vector.types.pojo.Field; import org.apache.arrow.vector.types.pojo.Schema; import org.apache.arrow.vector.util.DictionaryUtility; +import org.apache.arrow.vector.util.ValueVectorUtility; import org.apache.arrow.vector.util.VectorBatchAppender; /** Abstract class to read Schema and ArrowRecordBatches. */ @@ -48,6 +49,8 @@ public abstract class ArrowReader implements DictionaryProvider, AutoCloseable { private final CompressionCodec.Factory compressionFactory; + private boolean validateBatches = false; + protected ArrowReader(BufferAllocator allocator) { this(allocator, CompressionCodec.Factory.INSTANCE); } @@ -57,6 +60,34 @@ protected ArrowReader(BufferAllocator allocator, CompressionCodec.Factory compre this.compressionFactory = compressionFactory; } + /** + * Sets whether every record batch and dictionary batch is checked with {@link + * ValueVectorUtility#validateFull} once it is loaded. This is off by default, because the check + * reads every value of the batch. + * + *

Turn it on when the data comes from an untrusted source. The IPC format does not guarantee + * that offsets, views or run ends stay within the buffers they point into, and reading such a + * batch can read past the end of a buffer, without bounds checks when {@code + * arrow.enable_unsafe_memory_access} is set. With this option, a batch that fails the check is + * cleared and the load throws {@link + * org.apache.arrow.vector.validate.ValidateUtil.ValidateException}. + * + * @param validateBatches whether to check each loaded batch + */ + public void setValidateBatches(boolean validateBatches) { + this.validateBatches = validateBatches; + } + + /** + * Returns whether every loaded batch is checked with {@link ValueVectorUtility#validateFull}. + * + * @return true if loaded batches are checked + * @see #setValidateBatches(boolean) + */ + public boolean isValidateBatches() { + return validateBatches; + } + /** * Returns the vector schema root. This will be loaded with new values on every call to * loadNextBatch. @@ -214,6 +245,14 @@ protected void loadRecordBatch(ArrowRecordBatch batch) { } finally { batch.close(); } + if (validateBatches) { + try { + ValueVectorUtility.validateFull(root); + } catch (RuntimeException e) { + root.clear(); + throw e; + } + } } /** @@ -250,5 +289,15 @@ private void load(ArrowDictionaryBatch dictionaryBatch, FieldVector vector) { } finally { dictionaryBatch.close(); } + // A delta is checked before it is appended, so that an invalid delta leaves the dictionary + // unchanged + if (validateBatches) { + try { + ValueVectorUtility.validateFull(vector); + } catch (RuntimeException e) { + vector.clear(); + throw e; + } + } } } diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java index 5cfe64b14e..3f89478882 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorBufferVisitor.java @@ -34,7 +34,9 @@ import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; +import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.UnionVector; @@ -111,6 +113,15 @@ private void validateDataBuffer(ValueVector vector, long minCapacity) { dataBuffer.capacity()); } + private void validateSizeBuffer(ArrowBuf sizeBuffer, long minCapacity) { + validateOrThrow(sizeBuffer != null, "The size buffer is null."); + validateOrThrow( + sizeBuffer.capacity() >= minCapacity, + "Not enough capacity for the size buffer. Minimum capacity %s, actual capacity %s.", + minCapacity, + sizeBuffer.capacity()); + } + private void validateTypeBuffer(ArrowBuf typeBuf, long minCapacity) { validateOrThrow(typeBuf != null, "The type buffer is null."); validateOrThrow( @@ -248,6 +259,34 @@ public Void visit(LargeListVector vector, Void value) { return null; } + @Override + public Void visit(ListViewVector vector, Void value) { + int valueCount = vector.getValueCount(); + validateVectorCommon(vector); + validateValidityBuffer(vector, valueCount); + validateOffsetBuffer(vector, (long) valueCount * ListViewVector.OFFSET_WIDTH); + validateSizeBuffer(vector.getSizeBuffer(), (long) valueCount * ListViewVector.SIZE_WIDTH); + FieldVector dataVector = vector.getDataVector(); + if (dataVector != null) { + dataVector.accept(this, null); + } + return null; + } + + @Override + public Void visit(LargeListViewVector vector, Void value) { + int valueCount = vector.getValueCount(); + validateVectorCommon(vector); + validateValidityBuffer(vector, valueCount); + validateOffsetBuffer(vector, (long) valueCount * LargeListViewVector.OFFSET_WIDTH); + validateSizeBuffer(vector.getSizeBuffer(), (long) valueCount * LargeListViewVector.SIZE_WIDTH); + FieldVector dataVector = vector.getDataVector(); + if (dataVector != null) { + dataVector.accept(this, null); + } + return null; + } + @Override public Void visit(NonNullableStructVector vector, Void value) { int valueCount = vector.getValueCount(); diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java index 9da8cc813e..355229d744 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorDataVisitor.java @@ -18,20 +18,26 @@ import static org.apache.arrow.vector.validate.ValidateUtil.validateOrThrow; +import java.util.List; import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.vector.BaseFixedWidthVector; +import org.apache.arrow.vector.BaseIntVector; import org.apache.arrow.vector.BaseLargeVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthVector; import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.ExtensionTypeVector; +import org.apache.arrow.vector.FieldVector; import org.apache.arrow.vector.NullVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.compare.VectorVisitor; import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; +import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; import org.apache.arrow.vector.complex.NonNullableStructVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.UnionVector; /** Utility for validating vector data. */ @@ -121,10 +127,48 @@ public Void visit(BaseLargeVariableWidthVector vector, Void value) { @Override public Void visit(BaseVariableWidthViewVector vector, Void value) { + validateViews(vector); vector.validateScalars(); return null; } + private void validateViews(BaseVariableWidthViewVector vector) { + ArrowBuf viewBuffer = vector.getDataBuffer(); + List dataBuffers = vector.getDataBuffers(); + for (int i = 0; i < vector.getValueCount(); i++) { + if (vector.isNull(i)) { + continue; + } + long start = (long) i * BaseVariableWidthViewVector.ELEMENT_SIZE; + int length = viewBuffer.getInt(start); + validateOrThrow(length >= 0, "The view at position %s has a negative length: %s.", i, length); + if (length > BaseVariableWidthViewVector.INLINE_SIZE) { + long bufferIndexStart = + start + + BaseVariableWidthViewVector.LENGTH_WIDTH + + BaseVariableWidthViewVector.PREFIX_WIDTH; + int bufferIndex = viewBuffer.getInt(bufferIndexStart); + int offset = + viewBuffer.getInt(bufferIndexStart + BaseVariableWidthViewVector.BUF_INDEX_WIDTH); + validateOrThrow( + bufferIndex >= 0 && bufferIndex < dataBuffers.size(), + "The view at position %s refers to data buffer %s, but there are %s data buffers.", + i, + bufferIndex, + dataBuffers.size()); + long capacity = dataBuffers.get(bufferIndex).capacity(); + validateOrThrow( + offset >= 0 && (long) offset + length <= capacity, + "The view at position %s (offset %s, length %s) is outside data buffer %s of %s bytes.", + i, + offset, + length, + bufferIndex, + capacity); + } + } + } + @Override public Void visit(ListVector vector, Void value) { validateOffsetBuffer(vector, vector.getValueCount()); @@ -137,7 +181,7 @@ public Void visit(ListVector vector, Void value) { @Override public Void visit(FixedSizeListVector vector, Void value) { - validateOffsetBuffer(vector, vector.getValueCount()); + // A fixed size list has no offset buffer, and the buffer validation checks the inner length ValueVector innerVector = vector.getDataVector(); if (innerVector != null) { innerVector.accept(this, null); @@ -155,6 +199,91 @@ public Void visit(LargeListVector vector, Void value) { return null; } + @Override + public Void visit(ListViewVector vector, Void value) { + ValueVector innerVector = vector.getDataVector(); + int innerValueCount = innerVector == null ? 0 : innerVector.getValueCount(); + for (int i = 0; i < vector.getValueCount(); i++) { + if (vector.isNull(i)) { + continue; + } + int offset = vector.getOffsetBuffer().getInt((long) i * ListViewVector.OFFSET_WIDTH); + int size = vector.getSizeBuffer().getInt((long) i * ListViewVector.SIZE_WIDTH); + validateListView(i, offset, size, innerValueCount); + } + if (innerVector != null) { + innerVector.accept(this, null); + } + return null; + } + + @Override + public Void visit(LargeListViewVector vector, Void value) { + ValueVector innerVector = vector.getDataVector(); + int innerValueCount = innerVector == null ? 0 : innerVector.getValueCount(); + for (int i = 0; i < vector.getValueCount(); i++) { + if (vector.isNull(i)) { + continue; + } + long offset = vector.getOffsetBuffer().getLong((long) i * LargeListViewVector.OFFSET_WIDTH); + long size = vector.getSizeBuffer().getLong((long) i * LargeListViewVector.SIZE_WIDTH); + validateListView(i, offset, size, innerValueCount); + } + if (innerVector != null) { + innerVector.accept(this, null); + } + return null; + } + + private void validateListView(int index, long offset, long size, int innerValueCount) { + validateOrThrow( + size >= 0, "The size at position %s of the list view is negative: %s.", index, size); + validateOrThrow( + size == 0 || (offset >= 0 && offset <= innerValueCount - size), + "The list view at position %s (offset %s, size %s) is outside the inner vector of %s" + + " values.", + index, + offset, + size, + innerValueCount); + } + + @Override + public Void visit(RunEndEncodedVector vector, Void value) { + FieldVector runEndsVector = vector.getRunEndsVector(); + FieldVector valuesVector = vector.getValuesVector(); + if (runEndsVector != null) { + int runCount = runEndsVector.getValueCount(); + // Run ends are positive and strictly increasing, and each run has a value. Before its + // children are initialized, the run ends vector is empty and not an integer vector. + if (runCount > 0) { + BaseIntVector runEnds = (BaseIntVector) runEndsVector; + long previousEnd = 0; + for (int i = 0; i < runCount; i++) { + long end = runEnds.getValueAsLong(i); + validateOrThrow( + end > previousEnd, + "The run end at position %s (%s) is not greater than the previous run end (%s).", + i, + end, + previousEnd); + previousEnd = end; + } + int valueCount = valuesVector == null ? 0 : valuesVector.getValueCount(); + validateOrThrow( + valueCount >= runCount, + "The values vector has %s values, fewer than the %s runs.", + valueCount, + runCount); + } + runEndsVector.accept(this, null); + } + if (valuesVector != null) { + valuesVector.accept(this, null); + } + return null; + } + @Override public Void visit(NonNullableStructVector vector, Void value) { for (ValueVector subVector : vector.getChildrenFromFields()) { @@ -182,7 +311,12 @@ public Void visit(DenseUnionVector vector, Void value) { byte typeId = vector.getTypeId(i); ValueVector subVector = vector.getVectorByType(typeId); validateOrThrow( - offset < subVector.getValueCount(), + subVector != null, + "Dense union vector type id %s at position %s has no sub-vector.", + typeId, + i); + validateOrThrow( + offset >= 0 && offset < subVector.getValueCount(), "Dense union vector offset exceeds sub-vector boundary. Vector offset %s, sub vector size %s", offset, subVector.getValueCount()); diff --git a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java index 395852ef79..210dc34766 100644 --- a/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java +++ b/vector/src/main/java/org/apache/arrow/vector/validate/ValidateVectorTypeVisitor.java @@ -67,7 +67,10 @@ import org.apache.arrow.vector.complex.DenseUnionVector; import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; +import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.MapVector; import org.apache.arrow.vector.complex.NonNullableStructVector; import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.UnionVector; @@ -392,7 +395,9 @@ public Void visit(BaseVariableWidthViewVector vector, Void value) { @Override public Void visit(ListVector vector, Void value) { - validateVectorCommon(vector, ArrowType.List.class); + // A MapVector is a ListVector of type Map + validateVectorCommon( + vector, vector instanceof MapVector ? ArrowType.Map.class : ArrowType.List.class); ValueVector innerVector = vector.getDataVector(); if (innerVector != null) { innerVector.accept(this, null); @@ -428,6 +433,26 @@ public Void visit(LargeListVector vector, Void value) { return null; } + @Override + public Void visit(ListViewVector vector, Void value) { + validateVectorCommon(vector, ArrowType.ListView.class); + ValueVector innerVector = vector.getDataVector(); + if (innerVector != null) { + innerVector.accept(this, null); + } + return null; + } + + @Override + public Void visit(LargeListViewVector vector, Void value) { + validateVectorCommon(vector, ArrowType.LargeListView.class); + ValueVector innerVector = vector.getDataVector(); + if (innerVector != null) { + innerVector.accept(this, null); + } + return null; + } + @Override public Void visit(NonNullableStructVector vector, Void value) { validateVectorCommon(vector, ArrowType.Struct.class); diff --git a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java index b66d59e04d..de786f7895 100644 --- a/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java +++ b/vector/src/test/java/org/apache/arrow/vector/TestVariableWidthViewVector.java @@ -2934,8 +2934,9 @@ public void testValidateInvalidOffsets() { offsets.setInt(12, 1024); vector.setValueCount(1); vector.setIndexDefined(0); - var e = assertThrows(IndexOutOfBoundsException.class, vector::validateFull); - assertTrue(e.getMessage().contains("index: 1024")); + // The view is checked against the data buffer before the value is read + var e = assertThrows(ValidateUtil.ValidateException.class, vector::validateFull); + assertTrue(e.getMessage().contains("(offset 1024, length 64) is outside data buffer 0")); } } } diff --git a/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderValidation.java b/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderValidation.java new file mode 100644 index 0000000000..27df34bcd3 --- /dev/null +++ b/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowReaderValidation.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.arrow.vector.ipc; + +import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.channels.Channels; +import java.util.Arrays; +import java.util.Collections; +import org.apache.arrow.memory.ArrowBuf; +import org.apache.arrow.memory.BufferAllocator; +import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.ViewVarCharVector; +import org.apache.arrow.vector.complex.LargeListViewVector; +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; +import org.apache.arrow.vector.ipc.message.ArrowDictionaryBatch; +import org.apache.arrow.vector.ipc.message.ArrowFieldNode; +import org.apache.arrow.vector.ipc.message.ArrowRecordBatch; +import org.apache.arrow.vector.ipc.message.IpcOption; +import org.apache.arrow.vector.ipc.message.MessageSerializer; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.DictionaryEncoding; +import org.apache.arrow.vector.types.pojo.Field; +import org.apache.arrow.vector.types.pojo.FieldType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.validate.ValidateUtil; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestArrowReaderValidation { + + private BufferAllocator allocator; + + @BeforeEach + public void init() { + allocator = new RootAllocator(Long.MAX_VALUE); + } + + @AfterEach + public void terminate() { + allocator.close(); + } + + @Test + public void testValidBatchPassesValidation() throws Exception { + try (ViewVarCharVector views = new ViewVarCharVector("views", allocator); + ListViewVector listViews = ListViewVector.empty("listViews", allocator); + LargeListViewVector largeListViews = + LargeListViewVector.empty("largeListViews", allocator); + RunEndEncodedVector runs = RunEndEncodedVector.empty("runs", allocator)) { + setVector(views, "a", "a string longer than the inline size", null); + setVector(listViews, Arrays.asList(1, 2), Collections.emptyList(), Arrays.asList(3)); + setVector(largeListViews, Arrays.asList(1), Arrays.asList(2, 3), Collections.emptyList()); + setVector(runs, Arrays.asList(1, 3), Arrays.asList(10, 20)); + VectorSchemaRoot root = VectorSchemaRoot.of(views, listViews, largeListViews, runs); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (ArrowStreamWriter writer = new ArrowStreamWriter(root, null, out)) { + writer.writeBatch(); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(out.toByteArray()), allocator)) { + reader.setValidateBatches(true); + assertTrue(reader.loadNextBatch()); + assertEquals(root.contentToTSVString(), reader.getVectorSchemaRoot().contentToTSVString()); + assertFalse(reader.loadNextBatch()); + } + } + } + + @Test + public void testInvalidOffsetsRejected() throws Exception { + Schema schema = + new Schema(Collections.singletonList(Field.nullable("s", ArrowType.Utf8.INSTANCE))); + byte[] stream; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WriteChannel channel = new WriteChannel(Channels.newChannel(out)); + MessageSerializer.serialize(channel, schema); + try (ArrowRecordBatch batch = recordBatchWithInvalidOffsets()) { + MessageSerializer.serialize(channel, batch); + } + ArrowStreamWriter.writeEndOfStream(channel, IpcOption.DEFAULT); + stream = out.toByteArray(); + + // Not checked by default + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(stream), allocator)) { + assertTrue(reader.loadNextBatch()); + } + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(stream), allocator)) { + reader.setValidateBatches(true); + assertThrows(ValidateUtil.ValidateException.class, reader::loadNextBatch); + assertEquals(0, reader.getVectorSchemaRoot().getRowCount()); + } + } + + @Test + public void testInvalidDictionaryRejected() throws Exception { + DictionaryEncoding encoding = new DictionaryEncoding(1L, false, new ArrowType.Int(32, true)); + Schema schema = + new Schema( + Collections.singletonList( + new Field( + "s", + new FieldType(true, ArrowType.Utf8.INSTANCE, encoding), + Collections.emptyList()))); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + WriteChannel channel = new WriteChannel(Channels.newChannel(out)); + MessageSerializer.serialize(channel, schema); + try (ArrowDictionaryBatch batch = + new ArrowDictionaryBatch(1L, recordBatchWithInvalidOffsets())) { + MessageSerializer.serialize(channel, batch); + } + ArrowStreamWriter.writeEndOfStream(channel, IpcOption.DEFAULT); + + try (ArrowStreamReader reader = + new ArrowStreamReader(new ByteArrayInputStream(out.toByteArray()), allocator)) { + reader.setValidateBatches(true); + assertThrows(ValidateUtil.ValidateException.class, reader::loadNextBatch); + assertEquals(0, reader.lookup(1L).getVector().getValueCount()); + } + } + + /** + * Builds a batch with three Utf8 values whose second offset is past the end of the 8 byte data + * buffer. The last offset is within the buffer, so only a full validation finds it. + */ + private ArrowRecordBatch recordBatchWithInvalidOffsets() throws IOException { + int[] offsets = {0, 1_000_000, 2, 4}; + try (ArrowBuf validity = allocator.buffer(8); + ArrowBuf offsetBuffer = allocator.buffer(offsets.length * 4L); + ArrowBuf data = allocator.buffer(8)) { + validity.setOne(0, 8); + validity.writerIndex(8); + for (int i = 0; i < offsets.length; i++) { + offsetBuffer.setInt(i * 4L, offsets[i]); + } + offsetBuffer.writerIndex(offsets.length * 4L); + data.setZero(0, 8); + data.writerIndex(8); + return new ArrowRecordBatch( + 3, + Collections.singletonList(new ArrowFieldNode(3, 0)), + Arrays.asList(validity, offsetBuffer, data)); + } + } +} diff --git a/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java b/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java index 65a3791dd4..a666cf2378 100644 --- a/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java +++ b/vector/src/test/java/org/apache/arrow/vector/ipc/TestRoundTrip.java @@ -711,6 +711,9 @@ private void roundTrip( ArrowFileReader fileReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator); ArrowStreamReader streamReader = new ArrowStreamReader(inputStream, readerAllocator)) { + // Check every batch read back, so that the full validation is run on valid data of each type + fileReader.setValidateBatches(true); + streamReader.setValidateBatches(true); fileValidator.accept(fileReader); streamValidator.accept(streamReader); assertEquals(writeOption.metadataVersion, fileReader.getFooter().getMetadataVersion()); diff --git a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java index 849fe6d667..52775b467d 100644 --- a/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java +++ b/vector/src/test/java/org/apache/arrow/vector/testing/ValueVectorDataPopulator.java @@ -792,7 +792,7 @@ public static void setVector(LargeListViewVector vector, List... values for (int i = 0; i < values.length; i++) { vector .getOffsetBuffer() - .setInt((long) i * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH, curPos); + .setLong((long) i * BaseLargeRepeatedValueViewVector.OFFSET_WIDTH, curPos); if (values[i] == null) { BitVectorHelper.unsetBit(vector.getValidityBuffer(), i); } else { @@ -804,7 +804,7 @@ public static void setVector(LargeListViewVector vector, List... values } vector .getSizeBuffer() - .setInt((long) i * BaseRepeatedValueViewVector.SIZE_WIDTH, values[i].size()); + .setLong((long) i * BaseLargeRepeatedValueViewVector.SIZE_WIDTH, values[i].size()); } dataVector.setValueCount(curPos); vector.setValueCount(values.length); diff --git a/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java b/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java index 6993fde8fa..cb76453303 100644 --- a/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java +++ b/vector/src/test/java/org/apache/arrow/vector/validate/TestValidateVectorFull.java @@ -27,6 +27,7 @@ import org.apache.arrow.memory.ArrowBuf; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; +import org.apache.arrow.vector.BaseVariableWidthViewVector; import org.apache.arrow.vector.Decimal256Vector; import org.apache.arrow.vector.DecimalVector; import org.apache.arrow.vector.Float4Vector; @@ -34,9 +35,14 @@ import org.apache.arrow.vector.LargeVarCharVector; import org.apache.arrow.vector.ValueVector; import org.apache.arrow.vector.VarCharVector; +import org.apache.arrow.vector.ViewVarCharVector; import org.apache.arrow.vector.complex.DenseUnionVector; +import org.apache.arrow.vector.complex.FixedSizeListVector; import org.apache.arrow.vector.complex.LargeListVector; +import org.apache.arrow.vector.complex.LargeListViewVector; import org.apache.arrow.vector.complex.ListVector; +import org.apache.arrow.vector.complex.ListViewVector; +import org.apache.arrow.vector.complex.RunEndEncodedVector; import org.apache.arrow.vector.complex.StructVector; import org.apache.arrow.vector.complex.UnionVector; import org.apache.arrow.vector.holders.NullableFloat4Holder; @@ -209,6 +215,111 @@ public void testUnionVector() { } } + @Test + public void testFixedSizeListVector() { + try (final FixedSizeListVector vector = FixedSizeListVector.empty("v", 2, allocator)) { + validateFull(vector); + setVector(vector, Arrays.asList(1, 2), Arrays.asList(3, 4)); + validateFull(vector); + } + } + + @Test + public void testListViewVector() { + try (final ListViewVector vector = ListViewVector.empty("v", allocator)) { + validateFull(vector); + setVector(vector, Arrays.asList(1, 2, 3), Arrays.asList(4, 5)); + validateFull(vector); + + // The second list view (offset 3) now ends past the 5 values of the inner vector + vector.getSizeBuffer().setInt(ListViewVector.SIZE_WIDTH, 3); + + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("is outside the inner vector"), e.getMessage()); + } + } + + @Test + public void testLargeListViewVector() { + try (final LargeListViewVector vector = LargeListViewVector.empty("v", allocator)) { + validateFull(vector); + setVector(vector, Arrays.asList(1, 2, 3), Arrays.asList(4, 5)); + validateFull(vector); + + vector.getOffsetBuffer().setLong(LargeListViewVector.OFFSET_WIDTH, -1); + + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("is outside the inner vector"), e.getMessage()); + } + } + + @Test + public void testViewVarCharVector() { + try (final ViewVarCharVector vector = new ViewVarCharVector("v", allocator)) { + validateFull(vector); + setVector(vector, "short", "a string longer than the inline size"); + validateFull(vector); + + ArrowBuf views = vector.getDataBuffer(); + long bufferIndex = + BaseVariableWidthViewVector.ELEMENT_SIZE + + BaseVariableWidthViewVector.LENGTH_WIDTH + + BaseVariableWidthViewVector.PREFIX_WIDTH; + views.setInt(bufferIndex, 1); + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("refers to data buffer 1"), e.getMessage()); + + views.setInt(bufferIndex, 0); + views.setInt(bufferIndex + BaseVariableWidthViewVector.BUF_INDEX_WIDTH, 1_000_000); + e = assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("is outside data buffer 0"), e.getMessage()); + } + } + + @Test + public void testRunEndEncodedVector() { + try (final RunEndEncodedVector vector = RunEndEncodedVector.empty("v", allocator)) { + validateFull(vector); + setVector(vector, Arrays.asList(1, 3), Arrays.asList(10, 20)); + validateFull(vector); + + IntVector runEnds = (IntVector) vector.getRunEndsVector(); + runEnds.set(0, 3); + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("is not greater than the previous run end")); + + runEnds.set(0, 1); + vector.getValuesVector().setValueCount(1); + e = assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("fewer than the 2 runs"), e.getMessage()); + } + } + + @Test + public void testDenseUnionVectorNegativeOffset() { + try (final DenseUnionVector vector = DenseUnionVector.empty("union", allocator)) { + final NullableFloat4Holder float4Holder = new NullableFloat4Holder(); + float4Holder.value = 1.01f; + float4Holder.isSet = 1; + byte float4TypeId = + vector.registerNewTypeId(Field.nullable("", Types.MinorType.FLOAT4.getType())); + vector.setTypeId(0, float4TypeId); + vector.setSafe(0, float4Holder); + vector.setValueCount(1); + validateFull(vector); + + vector.getOffsetBuffer().setInt(0, -1); + + ValidateUtil.ValidateException e = + assertThrows(ValidateUtil.ValidateException.class, () -> validateFull(vector)); + assertTrue(e.getMessage().contains("Dense union vector offset exceeds sub-vector boundary")); + } + } + @Test public void testDenseUnionVector() { try (final DenseUnionVector vector = DenseUnionVector.empty("union", allocator)) {