Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
9ee42ed
Define the VECTOR schema contract
divjotarora Oct 22, 2026
1c8d398
Preserve metadata fallback for VECTOR elements
divjotarora Oct 22, 2026
d7d8da6
Enforce VECTOR validity during record writes
divjotarora Oct 22, 2026
8a44a99
Validate stored VECTOR boundaries during record reads
divjotarora Oct 22, 2026
4f95e9c
Reject contradictory VECTOR counts in selected chunks
divjotarora Oct 22, 2026
5e5daec
Read VECTOR files as Avro arrays
divjotarora Oct 22, 2026
1b31654
Map VECTOR files through existing model LIST paths
divjotarora Oct 22, 2026
36e534d
Cover VECTOR file contracts and malformed boundaries
divjotarora Oct 22, 2026
99ef065
Reject invalid VECTOR adapter interpretations
divjotarora Oct 22, 2026
4b1e872
Consolidate VECTOR schema and count validation
divjotarora Oct 22, 2026
5bdd8f1
Keep distinct VECTOR validation regressions
divjotarora Oct 22, 2026
0b4bba9
Use fixed VECTOR resources for focused file coverage
divjotarora Oct 22, 2026
5565d64
GH-3877: Align vector support with FINITE_VECTOR proposal
divjotarora Nov 2, 2026
974f3fb
GH-3877: Address FINITE_VECTOR schema and adapter review
divjotarora Nov 2, 2026
869eb6c
GH-3877: Validate FINITE_VECTOR schemas in the group builder
divjotarora Nov 4, 2026
362317e
GH-3877: Address FINITE_VECTOR validation and compatibility review
divjotarora Nov 11, 2026
c42087c
GH-3877: Remove per-value FINITE_VECTOR validation
divjotarora Nov 11, 2026
a7d8129
GH-3877: Simplify FINITE_VECTOR adapters and tests
divjotarora Nov 11, 2026
6493ae2
GH-3877: Validate FINITE_VECTOR lengths while reading records
divjotarora Nov 16, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ class List3Levels {
* @param list the Parquet List
*/
public List3Levels(GroupType list) {
if (list.getLogicalTypeAnnotation() != LogicalTypeAnnotation.listType()
LogicalTypeAnnotation logicalType = list.getLogicalTypeAnnotation();
if (!(logicalType instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation)
|| list.getFields().size() != 1) {
throw new IllegalArgumentException("invalid list type: " + list);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -486,6 +486,21 @@ public Optional<TypeMapping> visit(
return of(new ListTypeMapping(arrowField, list3Levels, child));
}

@Override
public Optional<TypeMapping> visit(
LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation vectorLogicalType) {
List3Levels list3Levels = new List3Levels(type);
Type element = list3Levels.getElement();
TypeMapping child = fromParquet(element, null, element.getRepetition());
ArrowType.FixedSizeList fixedSizeList =
new ArrowType.FixedSizeList(vectorLogicalType.getNumElements());

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice, this maps to FixedSizeList now 👍 The other direction still loses the length: fromArrow maps ArrowType.FixedSizeList to a plain LIST (createListTypeMapping()), so FINITE_VECTOR → Arrow → Parquet gives a LIST. Should we map it back to FINITE_VECTOR(listSize) when the child is a non-nullable supported primitive, or do we keep that for a follow-up?

Field arrowField = new Field(
name,
new FieldType(type.isRepetition(OPTIONAL), fixedSizeList, null),
Collections.singletonList(child.getArrowField()));
return of(new ListTypeMapping(arrowField, list3Levels, child));
}

@Override
public Optional<TypeMapping> visit(
LogicalTypeAnnotation.MapLogicalTypeAnnotation mapLogicalType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,34 @@
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.apache.parquet.schema.Types;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;

/**
* @see SchemaConverter
*/
public class TestSchemaConverter {
@ParameterizedTest
@ValueSource(strings = {"required", "optional"})
public void testVectorAsFixedSizeList(String repetition) {
String children = "repeated group list { required float element; }";
MessageType vector = MessageTypeParser.parseMessageType(
"message m { " + repetition + " group v (FINITE_VECTOR(5)) { " + children + " } }");
SchemaMapping mapping = converter.fromParquet(vector);
Field field = mapping.getArrowSchema().findField("v");
assertThat(field.getType()).isEqualTo(new ArrowType.FixedSizeList(5));
assertThat(field.isNullable()).isEqualTo(repetition.equals("optional"));
assertThat(field.getChildren()).hasSize(1);
assertThat(field.getChildren().get(0).getType())
.isEqualTo(new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE));
assertThat(field.getChildren().get(0).isNullable()).isFalse();
assertThat(mapping.getParquetSchema()).isEqualTo(vector);
assertThat(converter.map(mapping.getArrowSchema(), vector).getParquetSchema())
.isEqualTo(vector);
}

private static Field field(String name, boolean nullable, ArrowType type, Field... children) {
if (nullable) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ private static Boolean allListStructuresAreThreeLevel(Type type) {
return null;
}
GroupType group = type.asGroupType();
if (group.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
LogicalTypeAnnotation logicalType = group.getLogicalTypeAnnotation();
if (logicalType instanceof LogicalTypeAnnotation.ListLogicalTypeAnnotation) {
if (group.isRepetition(Type.Repetition.REPEATED) || group.getFieldCount() != 1) {
return false;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,7 @@ public Optional<Schema> visit(
if (!repeatedType.isRepetition(REPEATED)) {
throw new UnsupportedOperationException("Invalid list type " + parquetGroupType);
}
if (isElementType(repeatedType, parquetGroupType.getName())) {
if (isElementType(repeatedType, parquetGroupType)) {
// repeated element types are always required
return of(Schema.createArray(convertField(repeatedType, names)));
} else {
Expand Down Expand Up @@ -654,7 +654,10 @@ public Optional<LogicalType> visit(UUIDLogicalTypeAnnotation uuidLogicalType) {
* the same. This ensures that the LIST rules are followed when there is no
* schema and that a schema can be provided to override the default behavior.
*/
private boolean isElementType(Type repeatedType, String parentName) {
private boolean isElementType(Type repeatedType, GroupType parent) {
if (parent.getLogicalTypeAnnotation() instanceof LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation) {
return false;
}
return (
// can't be a synthetic layer because it would be invalid
repeatedType.isPrimitive()
Expand All @@ -663,7 +666,7 @@ private boolean isElementType(Type repeatedType, String parentName) {
||
// known patterns without the synthetic layer
repeatedType.getName().equals("array")
|| repeatedType.getName().equals(parentName + "_tuple")
|| repeatedType.getName().equals(parent.getName() + "_tuple")
||
// default assumption
assumeRepeatedIsListElement);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,20 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.parquet.DirectWriterTest;
import org.apache.parquet.example.data.Group;
import org.apache.parquet.example.data.simple.SimpleGroupFactory;
import org.apache.parquet.hadoop.ParquetFileReader;
import org.apache.parquet.hadoop.ParquetWriter;
import org.apache.parquet.hadoop.example.ExampleParquetWriter;
import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.apache.parquet.io.InvalidRecordException;
import org.apache.parquet.io.LocalInputFile;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.MessageTypeParser;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;

public class TestArrayCompatibility extends DirectWriterTest {

Expand Down Expand Up @@ -1610,6 +1617,53 @@ public void testAutoDetectTwoLevelRepeatedPrimitive() throws Exception {
assertReaderContains(new AvroParquetReader<>(newAutoDetectConf(), test), expectedSchema, expectedRecord);
}

@Test
public void testVectorWithoutAvroSchema(@TempDir java.nio.file.Path dir) throws IOException {
MessageType schema = MessageTypeParser.parseMessageType("message VectorWithoutAvroSchema {"
+ " required group v (FINITE_VECTOR(2)) { repeated group list { required float element; } }"
+ " optional group o (FINITE_VECTOR(1)) { repeated group list { required float element; } } }");
Path path = new Path(dir.resolve("vector.parquet").toUri());
SimpleGroupFactory factory = new SimpleGroupFactory(schema);
Group first = factory.newGroup();
first.addGroup("v").addGroup("list").append("element", 1.0f);
first.getGroup("v", 0).addGroup("list").append("element", 2.0f);
first.addGroup("o").addGroup("list").append("element", 9.0f);
Group second = factory.newGroup();
second.addGroup("v").addGroup("list").append("element", 3.0f);
second.getGroup("v", 0).addGroup("list").append("element", 4.0f);
try (ParquetWriter<Group> writer = ExampleParquetWriter.builder(path)
.withType(schema)
.withWriterVersion(org.apache.parquet.column.ParquetProperties.WriterVersion.PARQUET_1_0)
.withDictionaryEncoding(false)
.withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
.withValidation(false)
.build()) {
writer.write(first);
writer.write(second);
}
try (ParquetFileReader reader = ParquetFileReader.open(new LocalInputFile(dir.resolve("vector.parquet")))) {
assertThat(reader.getFileMetaData().getKeyValueMetaData())
.doesNotContainKeys(
AvroReadSupport.AVRO_SCHEMA_METADATA_KEY, AvroReadSupport.OLD_AVRO_SCHEMA_METADATA_KEY);
}
Schema expectedSchema = record(
"VectorWithoutAvroSchema",
field("v", array(primitive(Schema.Type.FLOAT))),
optionalField("o", array(primitive(Schema.Type.FLOAT))));
assertThat(new AvroSchemaConverter().convert(schema)).isEqualTo(expectedSchema);
for (boolean assumeRepeatedIsListElement : new boolean[] {true, false}) {
assertThat(new AvroSchemaConverter(assumeRepeatedIsListElement, false).convert(schema))
.isEqualTo(expectedSchema);
}
try (AvroParquetReader<GenericRecord> reader = autoDetectReader(path)) {
assertReaderContains(
reader,
expectedSchema,
instance(expectedSchema, "v", Arrays.asList(1.0f, 2.0f), "o", Arrays.asList(9.0f)),
instance(expectedSchema, "v", Arrays.asList(3.0f, 4.0f), "o", null));
}
}

private static Configuration newAutoDetectConf() {
Configuration conf = new Configuration();
conf.setBoolean(AvroReadSupport.AUTO_DETECT_LIST_STRUCTURE, true);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,18 @@ private void skipToMatch() {
State currentState = getState(0);
do {
ColumnReader columnReader = currentState.column;
int d = columnReader.getCurrentDefinitionLevel();

// currentLevel = depth + 1 at this point
// set the current value
if (columnReader.getCurrentDefinitionLevel() >= currentState.maxDefinitionLevel) {
if (d >= currentState.maxDefinitionLevel) {
columnReader.skip();
}
columnReader.consume();

// Based on repetition level work out next state to go to
int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
currentState.validateVector(d, nextR);
currentState = currentState.getNextState(nextR);
} while (currentState != null);
++recordsRead;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
import org.apache.parquet.io.api.PrimitiveConverter;
import org.apache.parquet.io.api.RecordConsumer;
import org.apache.parquet.io.api.RecordMaterializer;
import org.apache.parquet.schema.LogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
import org.apache.parquet.schema.MessageType;
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName;
import org.slf4j.Logger;
Expand Down Expand Up @@ -192,6 +194,8 @@ public static class State {
private Case[][][] caseLookup;
private List<Case> definedCases;
private List<Case> undefinedCases;
private final int vectorLength;
private long vectorSize;

private State(
int id,
Expand All @@ -213,6 +217,25 @@ private State(
this.primitiveField = fieldPath[fieldPath.length - 1];
this.indexFieldPath = primitiveColumnIO.getIndexFieldPath();
this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1];
LogicalTypeAnnotation logical = fieldPath.length < 3
? null
: primitiveColumnIO.getParent().getParent().getType().getLogicalTypeAnnotation();
this.vectorLength = logical instanceof FiniteVectorLogicalTypeAnnotation
? ((FiniteVectorLogicalTypeAnnotation) logical).getNumElements()
: 0;
}

void validateVector(int defLevel, int repLevel) {
if (vectorLength == 0) return;
if (defLevel == maxDefinitionLevel) ++vectorSize;
if (vectorSize > vectorLength
|| (repLevel < maxRepetitionLevel
&& (vectorSize > 0 || defLevel == maxDefinitionLevel - 1)
&& vectorSize != vectorLength)) {
throw new ParquetDecodingException("Invalid FINITE_VECTOR length for " + Arrays.toString(fieldPath)
+ ": expected " + vectorLength + " elements, got " + vectorSize);
}
if (repLevel < maxRepetitionLevel) vectorSize = 0;
}

public int getDepth(int definitionLevel) {
Expand Down Expand Up @@ -427,6 +450,7 @@ public T read() {
columnReader.consume();

int nextR = currentState.maxRepetitionLevel == 0 ? 0 : columnReader.getCurrentRepetitionLevel();
currentState.validateVector(d, nextR);
// level to go to close current groups
int next = currentState.nextLevel[nextR];
for (; currentLevel > next; currentLevel--) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ protected LogicalTypeAnnotation fromString(List<String> params) {
return listType();
}
},
FINITE_VECTOR {
@Override
protected LogicalTypeAnnotation fromString(List<String> params) {
Preconditions.checkArgument(params.size() == 1, "Expecting 1 parameter for FINITE_VECTOR: %s", params);
return finiteVectorType(Integer.parseInt(params.get(0)));
}
},
VARIANT {
@Override
protected LogicalTypeAnnotation fromString(List<String> params) {
Expand Down Expand Up @@ -314,6 +321,10 @@ public static ListLogicalTypeAnnotation listType() {
return ListLogicalTypeAnnotation.INSTANCE;
}

public static FiniteVectorLogicalTypeAnnotation finiteVectorType(int numElements) {
return new FiniteVectorLogicalTypeAnnotation(numElements);
}

public static VariantLogicalTypeAnnotation variantType(byte specVersion) {
return new VariantLogicalTypeAnnotation(specVersion);
}
Expand Down Expand Up @@ -503,7 +514,7 @@ LogicalTypeToken getType() {

@Override
public boolean equals(Object obj) {
return obj instanceof ListLogicalTypeAnnotation;
return obj instanceof ListLogicalTypeAnnotation && !(obj instanceof FiniteVectorLogicalTypeAnnotation);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: this way the parent needs to know about its subclass. How about:

Suggested change
return obj instanceof ListLogicalTypeAnnotation && !(obj instanceof FiniteVectorLogicalTypeAnnotation);
return obj != null && obj.getClass() == ListLogicalTypeAnnotation.class;

}

@Override
Expand All @@ -513,6 +524,45 @@ public int hashCode() {
}
}

public static class FiniteVectorLogicalTypeAnnotation extends ListLogicalTypeAnnotation {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The spec says every non-null vector MUST have exactly num_elements elements and every element MUST be finite. Since parquet-java doesn't enforce either (testVectorValuesAreNotValidated writes NaN/Inf), could we add a Javadoc here saying so? Then users know it's up to them to write valid vectors.

private final int numElements;

private FiniteVectorLogicalTypeAnnotation(int numElements) {
Preconditions.checkArgument(numElements > 0, "Invalid FINITE_VECTOR length: %s", numElements);
this.numElements = numElements;
}

public int getNumElements() {
return numElements;
}

@Override
public <T> Optional<T> accept(LogicalTypeAnnotationVisitor<T> visitor) {
return visitor.visit(this);
}

@Override
LogicalTypeToken getType() {
return LogicalTypeToken.FINITE_VECTOR;
}

@Override
protected String typeParametersAsString() {
return "(" + numElements + ")";
}

@Override
public boolean equals(Object obj) {
return obj instanceof FiniteVectorLogicalTypeAnnotation
&& numElements == ((FiniteVectorLogicalTypeAnnotation) obj).numElements;
}

@Override
public int hashCode() {
return 31 * getClass().hashCode() + numElements;
}
}

public static class EnumLogicalTypeAnnotation extends LogicalTypeAnnotation {
private static final EnumLogicalTypeAnnotation INSTANCE = new EnumLogicalTypeAnnotation();

Expand Down Expand Up @@ -1536,6 +1586,10 @@ default Optional<T> visit(ListLogicalTypeAnnotation listLogicalType) {
return empty();
}

default Optional<T> visit(FiniteVectorLogicalTypeAnnotation vectorLogicalType) {
return visit((ListLogicalTypeAnnotation) vectorLogicalType);
}
Comment thread
divjotarora marked this conversation as resolved.
Resolved

default Optional<T> visit(VariantLogicalTypeAnnotation variantLogicalType) {
return empty();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,53 @@
*/
package org.apache.parquet.schema;

import org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.FiniteVectorLogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.Float16LogicalTypeAnnotation;
import org.apache.parquet.schema.LogicalTypeAnnotation.IntLogicalTypeAnnotation;

public class TypeUtil {
static void checkFiniteVector(GroupType vector) {
if (vector.isRepetition(Type.Repetition.REPEATED) || vector.getFieldCount() != 1) {
throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + vector);
}
Type list = vector.getType(0);
if (list.isPrimitive()
|| !list.getName().equals("list")
|| !list.isRepetition(Type.Repetition.REPEATED)
|| list.getLogicalTypeAnnotation() != null
|| list.asGroupType().getFieldCount() != 1) {
throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + vector);
}
Type element = list.asGroupType().getType(0);
if (!element.isPrimitive()
|| !element.getName().equals("element")
|| !element.isRepetition(Type.Repetition.REQUIRED)) {
throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + vector);
}
LogicalTypeAnnotation logical = element.getLogicalTypeAnnotation();
boolean allowed = switch (element.asPrimitiveType().getPrimitiveTypeName()) {
case BOOLEAN, FLOAT, DOUBLE -> logical == null;
case INT32, INT64 ->
logical == null
|| logical instanceof IntLogicalTypeAnnotation
|| logical instanceof DecimalLogicalTypeAnnotation;
case FIXED_LEN_BYTE_ARRAY ->
logical instanceof Float16LogicalTypeAnnotation || logical instanceof DecimalLogicalTypeAnnotation;
default -> false;
};
if (!allowed) {
throw new InvalidSchemaException("Invalid FINITE_VECTOR schema: " + vector);
}
}

public static void checkValidWriteSchema(GroupType schema) {
schema.accept(new TypeVisitor() {
@Override
public void visit(GroupType groupType) {
if (groupType.getLogicalTypeAnnotation() instanceof FiniteVectorLogicalTypeAnnotation) {
checkFiniteVector(groupType);
}
if (groupType.getFieldCount() <= 0) {
throw new InvalidSchemaException("Cannot write a schema with an empty group: " + groupType);
}
Expand Down
Loading
Loading