diff --git a/parquet/internal/gen-go/parquet/parquet.go b/parquet/internal/gen-go/parquet/parquet.go index e6c06fc55..7dc1018f6 100644 --- a/parquet/internal/gen-go/parquet/parquet.go +++ b/parquet/internal/gen-go/parquet/parquet.go @@ -5043,6 +5043,148 @@ func (p *GeographyType) Validate() error { return nil } +// Finite fixed-length vector logical type annotation +// +// Annotates the outer group of a three-level structure that is the same as +// the one used for LIST-annotated groups. Each non-null vector MUST contain +// exactly num_elements elements. +// +// See LogicalTypes.md for details. +// +// Attributes: +// - NumElements +// +type FiniteVectorType struct { + NumElements int32 `thrift:"num_elements,1,required" db:"num_elements" json:"num_elements"` +} + +func NewFiniteVectorType() *FiniteVectorType { + return &FiniteVectorType{} +} + + + +func (p *FiniteVectorType) GetNumElements() int32 { + return p.NumElements +} + +func (p *FiniteVectorType) Read(ctx context.Context, iprot thrift.TProtocol) error { + if _, err := iprot.ReadStructBegin(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) + } + + var issetNumElements bool = false; + + for { + _, fieldTypeId, fieldId, err := iprot.ReadFieldBegin(ctx) + if err != nil { + return thrift.PrependError(fmt.Sprintf("%T field %d read error: ", p, fieldId), err) + } + if fieldTypeId == thrift.STOP { + break + } + switch fieldId { + case 1: + if fieldTypeId == thrift.I32 { + if err := p.ReadField1(ctx, iprot); err != nil { + return err + } + issetNumElements = true + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + default: + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } + if err := iprot.ReadFieldEnd(ctx); err != nil { + return err + } + } + if err := iprot.ReadStructEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T read struct end error: ", p), err) + } + if !issetNumElements{ + return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("Required field NumElements is not set")) + } + return nil +} + +func (p *FiniteVectorType) ReadField1(ctx context.Context, iprot thrift.TProtocol) error { + if v, err := iprot.ReadI32(ctx); err != nil { + return thrift.PrependError("error reading field 1: ", err) + } else { + p.NumElements = v + } + return nil +} + +func (p *FiniteVectorType) Write(ctx context.Context, oprot thrift.TProtocol) error { + if err := oprot.WriteStructBegin(ctx, "FiniteVectorType"); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write struct begin error: ", p), err) + } + if p != nil { + if err := p.writeField1(ctx, oprot); err != nil { return err } + } + if err := oprot.WriteFieldStop(ctx); err != nil { + return thrift.PrependError("write field stop error: ", err) + } + if err := oprot.WriteStructEnd(ctx); err != nil { + return thrift.PrependError("write struct stop error: ", err) + } + return nil +} + +func (p *FiniteVectorType) writeField1(ctx context.Context, oprot thrift.TProtocol) (err error) { + if err := oprot.WriteFieldBegin(ctx, "num_elements", thrift.I32, 1); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 1:num_elements: ", p), err) + } + if err := oprot.WriteI32(ctx, int32(p.NumElements)); err != nil { + return thrift.PrependError(fmt.Sprintf("%T.num_elements (1) field write error: ", p), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 1:num_elements: ", p), err) + } + return err +} + +func (p *FiniteVectorType) Equals(other *FiniteVectorType) bool { + if p == other { + return true + } else if p == nil || other == nil { + return false + } + if p.NumElements != other.NumElements { return false } + return true +} + +func (p *FiniteVectorType) String() string { + if p == nil { + return "" + } + return fmt.Sprintf("FiniteVectorType(%+v)", *p) +} + +func (p *FiniteVectorType) LogValue() slog.Value { + if p == nil { + return slog.AnyValue(nil) + } + v := thrift.SlogTStructWrapper{ + Type: "*parquet.FiniteVectorType", + Value: p, + } + return slog.AnyValue(v) +} + +var _ slog.LogValuer = (*FiniteVectorType)(nil) + +func (p *FiniteVectorType) Validate() error { + return nil +} + // LogicalType annotations to replace ConvertedType. // // To maintain compatibility, implementations using LogicalType for a @@ -5067,6 +5209,7 @@ func (p *GeographyType) Validate() error { // - VARIANT // - GEOMETRY // - GEOGRAPHY +// - FINITE_VECTOR // type LogicalType struct { STRING *StringType `thrift:"STRING,1" db:"STRING" json:"STRING,omitempty"` @@ -5087,6 +5230,8 @@ type LogicalType struct { VARIANT *VariantType `thrift:"VARIANT,16" db:"VARIANT" json:"VARIANT,omitempty"` GEOMETRY *GeometryType `thrift:"GEOMETRY,17" db:"GEOMETRY" json:"GEOMETRY,omitempty"` GEOGRAPHY *GeographyType `thrift:"GEOGRAPHY,18" db:"GEOGRAPHY" json:"GEOGRAPHY,omitempty"` + // unused field # 19 + FINITE_VECTOR *FiniteVectorType `thrift:"FINITE_VECTOR,20" db:"FINITE_VECTOR" json:"FINITE_VECTOR,omitempty"` } func NewLogicalType() *LogicalType { @@ -5246,6 +5391,15 @@ func (p *LogicalType) GetGEOGRAPHY() *GeographyType { return p.GEOGRAPHY } +var LogicalType_FINITE_VECTOR_DEFAULT *FiniteVectorType + +func (p *LogicalType) GetFINITE_VECTOR() *FiniteVectorType { + if !p.IsSetFINITE_VECTOR() { + return LogicalType_FINITE_VECTOR_DEFAULT + } + return p.FINITE_VECTOR +} + func (p *LogicalType) CountSetFieldsLogicalType() int { count := 0 if (p.IsSetSTRING()) { @@ -5299,6 +5453,9 @@ func (p *LogicalType) CountSetFieldsLogicalType() int { if (p.IsSetGEOGRAPHY()) { count++ } + if (p.IsSetFINITE_VECTOR()) { + count++ + } return count } @@ -5371,6 +5528,10 @@ func (p *LogicalType) IsSetGEOGRAPHY() bool { return p.GEOGRAPHY != nil } +func (p *LogicalType) IsSetFINITE_VECTOR() bool { + return p.FINITE_VECTOR != nil +} + func (p *LogicalType) Read(ctx context.Context, iprot thrift.TProtocol) error { if _, err := iprot.ReadStructBegin(ctx); err != nil { return thrift.PrependError(fmt.Sprintf("%T read error: ", p), err) @@ -5556,6 +5717,16 @@ func (p *LogicalType) Read(ctx context.Context, iprot thrift.TProtocol) error { return err } } + case 20: + if fieldTypeId == thrift.STRUCT { + if err := p.ReadField20(ctx, iprot); err != nil { + return err + } + } else { + if err := iprot.Skip(ctx, fieldTypeId); err != nil { + return err + } + } default: if err := iprot.Skip(ctx, fieldTypeId); err != nil { return err @@ -5707,6 +5878,14 @@ func (p *LogicalType) ReadField18(ctx context.Context, iprot thrift.TProtocol) e return nil } +func (p *LogicalType) ReadField20(ctx context.Context, iprot thrift.TProtocol) error { + p.FINITE_VECTOR = &FiniteVectorType{} + if err := p.FINITE_VECTOR.Read(ctx, iprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error reading struct: ", p.FINITE_VECTOR), err) + } + return nil +} + func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error { if c := p.CountSetFieldsLogicalType(); c != 1 { return thrift.NewTProtocolExceptionWithType(thrift.INVALID_DATA, fmt.Errorf("%T write union: exactly one field must be set (%d set)", p, c)) @@ -5732,6 +5911,7 @@ func (p *LogicalType) Write(ctx context.Context, oprot thrift.TProtocol) error { if err := p.writeField16(ctx, oprot); err != nil { return err } if err := p.writeField17(ctx, oprot); err != nil { return err } if err := p.writeField18(ctx, oprot); err != nil { return err } + if err := p.writeField20(ctx, oprot); err != nil { return err } } if err := oprot.WriteFieldStop(ctx); err != nil { return thrift.PrependError("write field stop error: ", err) @@ -5997,6 +6177,21 @@ func (p *LogicalType) writeField18(ctx context.Context, oprot thrift.TProtocol) return err } +func (p *LogicalType) writeField20(ctx context.Context, oprot thrift.TProtocol) (err error) { + if p.IsSetFINITE_VECTOR() { + if err := oprot.WriteFieldBegin(ctx, "FINITE_VECTOR", thrift.STRUCT, 20); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field begin error 20:FINITE_VECTOR: ", p), err) + } + if err := p.FINITE_VECTOR.Write(ctx, oprot); err != nil { + return thrift.PrependError(fmt.Sprintf("%T error writing struct: ", p.FINITE_VECTOR), err) + } + if err := oprot.WriteFieldEnd(ctx); err != nil { + return thrift.PrependError(fmt.Sprintf("%T write field end error 20:FINITE_VECTOR: ", p), err) + } + } + return err +} + func (p *LogicalType) Equals(other *LogicalType) bool { if p == other { return true @@ -6020,6 +6215,7 @@ func (p *LogicalType) Equals(other *LogicalType) bool { if !p.VARIANT.Equals(other.VARIANT) { return false } if !p.GEOMETRY.Equals(other.GEOMETRY) { return false } if !p.GEOGRAPHY.Equals(other.GEOGRAPHY) { return false } + if !p.FINITE_VECTOR.Equals(other.FINITE_VECTOR) { return false } return true } @@ -12928,6 +13124,8 @@ func (p *Int96TimestampOrder) Validate() error { // VARIANT - undefined // GEOMETRY - undefined // GEOGRAPHY - undefined +// FILE - undefined +// FINITE_VECTOR - undefined // // In the absence of logical types, the sort order is determined by the physical type: // BOOLEAN - false, true diff --git a/parquet/pqarrow/column_readers.go b/parquet/pqarrow/column_readers.go index 237df7d8c..d07ef85f1 100644 --- a/parquet/pqarrow/column_readers.go +++ b/parquet/pqarrow/column_readers.go @@ -487,7 +487,8 @@ func (lr *listReader) buildArray(lenBound int64) (*arrow.Chunked, error) { } validityIO := file.ValidityBitmapInputOutput{ReadUpperBound: lenBound} - if lr.field.Nullable { + // Required fixed-size lists can still be absent under a null struct. + if lr.field.Nullable || lr.field.Type.ID() == arrow.FIXED_SIZE_LIST { validityBuffer = memory.NewResizableBuffer(lr.rctx.mem) validityBuffer.Resize(int(bitutil.BytesForBits(lenBound))) defer validityBuffer.Release() @@ -594,7 +595,7 @@ func (lr *listReader) buildFixedSizeListArray(length int, offsets []int32, valid runCount := 0 previousValid := false for idx := 0; idx < length; idx++ { - valid := !lr.field.Nullable || bitutil.BitIsSet(validity, idx) + valid := bitutil.BitIsSet(validity, idx) if valid { if size := offsets[idx+1] - offsets[idx]; size != int32(listSize) { return nil, fmt.Errorf("expected all lists to be of size=%d, but index %d had size=%d", listSize, idx, size) @@ -619,10 +620,10 @@ func (lr *listReader) buildFixedSizeListArray(length int, offsets []int32, valid pieces := make([]arrow.Array, 0, runCount) defer func() { releaseArrays(pieces) }() for i := 0; i < length; { - valid := !lr.field.Nullable || bitutil.BitIsSet(validity, i) + valid := bitutil.BitIsSet(validity, i) end := i + 1 for end < length { - nextValid := !lr.field.Nullable || bitutil.BitIsSet(validity, end) + nextValid := bitutil.BitIsSet(validity, end) if nextValid != valid { break } @@ -652,10 +653,10 @@ func (lr *listReader) buildFixedSizeListArray(length int, offsets []int32, valid clear(indicesValidity.Bytes()) for i := 0; i < length; { - valid := !lr.field.Nullable || bitutil.BitIsSet(validity, i) + valid := bitutil.BitIsSet(validity, i) end := i + 1 for end < length { - nextValid := !lr.field.Nullable || bitutil.BitIsSet(validity, end) + nextValid := bitutil.BitIsSet(validity, end) if nextValid != valid { break } diff --git a/parquet/pqarrow/encode_arrow.go b/parquet/pqarrow/encode_arrow.go index cfa7b1454..2bebdb84e 100644 --- a/parquet/pqarrow/encode_arrow.go +++ b/parquet/pqarrow/encode_arrow.go @@ -191,7 +191,7 @@ type arrowColumnWriter struct { // // Using an arrow column writer is a convenience to avoid having to process the arrow array yourself // and determine the correct definition and repetition levels manually. -func newArrowColumnWriter(data *arrow.Chunked, offset, size int64, manifest *SchemaManifest, rgw file.RowGroupWriter, leafColIdx int) (arrowColumnWriter, error) { +func newArrowColumnWriter(data *arrow.Chunked, offset, size int64, manifest *SchemaManifest, rgw file.RowGroupWriter, leafColIdx int, writeFiniteVector bool) (arrowColumnWriter, error) { if data.Len() == 0 { return arrowColumnWriter{leafCount: calcLeafCount(data.DataType()), rgw: rgw}, nil } @@ -234,6 +234,18 @@ func newArrowColumnWriter(data *arrow.Chunked, offset, size int64, manifest *Sch } isNullable = nullableRoot(manifest, schemaField) + var vectorLeaves []bool + if writeFiniteVector { + for leaf := 0; leaf < leafCount; leaf++ { + if isFiniteVectorLeaf(manifest.descr.Column(leafColIdx + leaf).SchemaNode()) { + if vectorLeaves == nil { + vectorLeaves = make([]bool, leafCount) + } + vectorLeaves[leaf] = true + } + } + } + builders := make([]*multipathLevelBuilder, 0) releaseBuilders := func() { for _, bldr := range builders { @@ -250,7 +262,7 @@ func newArrowColumnWriter(data *arrow.Chunked, offset, size int64, manifest *Sch arrToWrite := array.NewSlice(chunk, chunkOffset, chunkOffset+chunkWriteSize) if arrToWrite.Len() > 0 { - bldr, err := newMultipathLevelBuilder(arrToWrite, isNullable) + bldr, err := buildMultipathLevels(arrToWrite, isNullable, vectorLeaves) arrToWrite.Release() if err != nil { releaseBuilders() @@ -349,6 +361,11 @@ func visitedLeafValues(res *multipathLevelResult, mem memory.Allocator) (arrow.A // Leaf_field_nullable indicates whether the leaf array is considered nullable // according to its schema in a Table or its parent array. func WriteArrowToColumn(ctx context.Context, cw file.ColumnChunkWriter, leafArr arrow.Array, defLevels, repLevels []int16, leafFieldNullable bool) error { + if isFiniteVectorLeaf(cw.Descr().SchemaNode()) { + if err := checkFiniteVectorValues(leafArr); err != nil { + return fmt.Errorf("FINITE_VECTOR column %q: %w", cw.Descr().Path(), err) + } + } // Leaf nulls are canonical when there is only a single null element after a list // and it is at the leaf. colLevelInfo := cw.LevelInfo() @@ -375,6 +392,48 @@ func WriteArrowToColumn(ctx context.Context, cw file.ColumnChunkWriter, leafArr return writeDenseArrow(arrCtx, cw, leafArr, defLevels, repLevels, maybeParentNulls) } +func isFiniteVectorLeaf(node schema.Node) bool { + list := node.Parent() + if list == nil || list.Parent() == nil { + return false + } + _, ok := list.Parent().LogicalType().(schema.FiniteVectorLogicalType) + return ok +} + +func checkFiniteVectorValues(values arrow.Array) error { + for values.DataType().ID() == arrow.EXTENSION { + values = values.(array.ExtensionArray).Storage() + } + if values.DataType().ID() == arrow.DICTIONARY { + return fmt.Errorf("%w: dictionary arrays are not supported for FINITE_VECTOR elements", arrow.ErrInvalid) + } + if values.NullN() > 0 { + return fmt.Errorf("%w: contains a null element", arrow.ErrInvalid) + } + switch typed := values.(type) { + case *array.Float16: + for _, value := range typed.Values() { + if value.IsNaN() || value.IsInf() { + return fmt.Errorf("%w: contains a non-finite element", arrow.ErrInvalid) + } + } + case *array.Float32: + for _, value := range typed.Values() { + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return fmt.Errorf("%w: contains a non-finite element", arrow.ErrInvalid) + } + } + case *array.Float64: + for _, value := range typed.Values() { + if math.IsNaN(value) || math.IsInf(value, 0) { + return fmt.Errorf("%w: contains a non-finite element", arrow.ErrInvalid) + } + } + } + return nil +} + type binaryarr interface { ValueOffsets() []int32 } diff --git a/parquet/pqarrow/file_writer.go b/parquet/pqarrow/file_writer.go index 4a96ecb62..48d7d07f3 100644 --- a/parquet/pqarrow/file_writer.go +++ b/parquet/pqarrow/file_writer.go @@ -525,7 +525,8 @@ func (fw *FileWriter) WriteColumnChunked(data *arrow.Chunked, offset, size int64 if fw.closed { return fmt.Errorf("invalid write call: FileWriter is already closed") } - acw, err := newArrowColumnWriter(data, offset, size, fw.manifest, fw.rgw, fw.colIdx) + acw, err := newArrowColumnWriter(data, offset, size, fw.manifest, fw.rgw, fw.colIdx, + fw.arrowProps.writeFiniteVector) if err != nil { return err } diff --git a/parquet/pqarrow/finite_vector_read_test.go b/parquet/pqarrow/finite_vector_read_test.go new file mode 100644 index 000000000..2e4abf9e0 --- /dev/null +++ b/parquet/pqarrow/finite_vector_read_test.go @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pqarrow_test + +import ( + "bytes" + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/pqarrow" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/stretchr/testify/require" +) + +func TestReadFiniteVectorWithoutArrowSchema(t *testing.T) { + for _, tc := range []struct { + name string + defs []int16 + reps []int16 + values []float32 + badSize bool + }{ + {"fixed size and null vector", []int16{2, 2, 2, 0}, []int16{0, 1, 1, 0}, []float32{1, 2, 3}, false}, + {"wrong size", []int16{2, 2}, []int16{0, 1}, []float32{1, 2}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + logical, err := schema.NewFiniteVectorLogicalType(3) + require.NoError(t, err) + element := schema.Must(schema.NewPrimitiveNodeLogical("element", parquet.Repetitions.Required, + nil, parquet.Types.Float, -1, -1)) + list := schema.Must(schema.NewGroupNode("list", parquet.Repetitions.Repeated, schema.FieldList{element}, -1)) + vector := schema.Must(schema.NewGroupNodeLogical("v", parquet.Repetitions.Optional, schema.FieldList{list}, logical, -1)) + root := schema.MustGroup(schema.NewGroupNode("schema", parquet.Repetitions.Required, schema.FieldList{vector}, -1)) + + var sink bytes.Buffer + writer := file.NewParquetWriter(&sink, root) + group, err := writer.AppendRowGroupChecked() + require.NoError(t, err) + column, err := group.NextColumn() + require.NoError(t, err) + _, err = column.(*file.Float32ColumnChunkWriter).WriteBatch(tc.values, tc.defs, tc.reps) + require.NoError(t, err) + require.NoError(t, group.Close()) + require.NoError(t, writer.Close()) + + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + got, err := pqarrow.ReadTable(context.Background(), bytes.NewReader(sink.Bytes()), nil, pqarrow.ArrowReadProperties{}, mem) + if tc.badSize { + require.ErrorContains(t, err, "expected all lists to be of size=3") + return + } + require.NoError(t, err) + defer got.Release() + wantType := arrow.FixedSizeListOfField(3, arrow.Field{Name: "element", Type: arrow.PrimitiveTypes.Float32}) + require.True(t, arrow.TypeEqual(wantType, got.Schema().Field(0).Type)) + want, _, err := array.FromJSON(mem, wantType, bytes.NewBufferString("[[1, 2, 3], null]")) + require.NoError(t, err) + defer want.Release() + require.True(t, array.Equal(want, got.Column(0).Data().Chunk(0))) + }) + } +} diff --git a/parquet/pqarrow/finite_vector_write_test.go b/parquet/pqarrow/finite_vector_write_test.go new file mode 100644 index 000000000..472add2c3 --- /dev/null +++ b/parquet/pqarrow/finite_vector_write_test.go @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !pqarrow_read_only + +package pqarrow_test + +import ( + "bytes" + "context" + "math" + "strings" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/file" + "github.com/apache/arrow-go/v18/parquet/pqarrow" + "github.com/apache/arrow-go/v18/parquet/schema" + "github.com/stretchr/testify/require" +) + +func TestWriteFiniteVector(t *testing.T) { + ctx := context.Background() + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + typ := arrow.FixedSizeListOf(3, arrow.PrimitiveTypes.Float32) + field := arrow.Field{Name: "v", Type: typ, Nullable: true} + sc := arrow.NewSchema([]arrow.Field{field}, nil) + write := func(values arrow.Array, opts ...pqarrow.WriterOption) ([]byte, error) { + col := arrow.NewColumnFromArr(field, values) + defer col.Release() + table := array.NewTable(sc, []arrow.Column{col}, int64(values.Len())) + defer table.Release() + var sink bytes.Buffer + opts = append(opts, pqarrow.WithAllocator(mem)) + err := pqarrow.WriteTable(table, &sink, 2, parquet.NewWriterProperties(parquet.WithAllocator(mem)), pqarrow.NewArrowWriterProperties(opts...)) + return sink.Bytes(), err + } + + values, _, err := array.FromJSON(mem, typ, strings.NewReader("[[1, 2, 3], null, [4, 5, 6]]")) + require.NoError(t, err) + defer values.Release() + for _, storeSchema := range []bool{false, true} { + opts := []pqarrow.WriterOption{pqarrow.WithFiniteVectorLogicalType()} + if storeSchema { + opts = append(opts, pqarrow.WithStoreSchema()) + } + encoded, err := write(values, opts...) + require.NoError(t, err) + reader, err := file.NewParquetReader(bytes.NewReader(encoded)) + require.NoError(t, err) + node := reader.MetaData().Schema.Root().Field(0) + logical, ok := node.LogicalType().(schema.FiniteVectorLogicalType) + require.True(t, ok) + require.Equal(t, int32(3), logical.NumElements()) + require.Equal(t, schema.ConvertedTypes.List, node.ConvertedType()) + require.NoError(t, reader.Close()) + + result, err := pqarrow.ReadTable(ctx, bytes.NewReader(encoded), parquet.NewReaderProperties(mem), pqarrow.ArrowReadProperties{}, mem) + require.NoError(t, err) + defer result.Release() + gotType := result.Schema().Field(0).Type.(*arrow.FixedSizeListType) + require.Equal(t, int32(3), gotType.Len()) + require.Equal(t, storeSchema, gotType.ElemField().Nullable) + got, err := array.Concatenate(result.Column(0).Data().Chunks(), mem) + require.NoError(t, err) + defer got.Release() + want, _, err := array.FromJSON(mem, gotType, strings.NewReader("[[1, 2, 3], null, [4, 5, 6]]")) + require.NoError(t, err) + defer want.Release() + require.True(t, array.Equal(want, got)) + } + + for _, tc := range []struct { + name string + bad float32 + valid bool + message string + }{ + {"null", 0, false, "contains a null element"}, + {"NaN", float32(math.NaN()), true, "contains a non-finite element"}, + {"infinity", float32(math.Inf(1)), true, "contains a non-finite element"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := array.NewFixedSizeListBuilder(mem, 3, arrow.PrimitiveTypes.Float32) + defer b.Release() + b.Append(true) + items := b.ValueBuilder().(*array.Float32Builder) + items.AppendValues([]float32{1, tc.bad, 3}, []bool{true, tc.valid, true}) + bad := b.NewArray() + defer bad.Release() + _, err := write(bad, pqarrow.WithFiniteVectorLogicalType()) + require.ErrorContains(t, err, tc.message) + _, err = write(bad) + require.NoError(t, err, "ordinary lists allow these values") + }) + } +} + +func TestFiniteVectorUnderNullStruct(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + vector := arrow.FixedSizeListOfField(3, arrow.Field{Name: "element", Type: arrow.PrimitiveTypes.Float32}) + typ := arrow.StructOf( + arrow.Field{Name: "score", Type: arrow.PrimitiveTypes.Int32}, + arrow.Field{Name: "v", Type: vector}, + ) + field := arrow.Field{Name: "player", Type: typ, Nullable: true} + values, _, err := array.FromJSON(mem, typ, strings.NewReader(`[{"score":1,"v":[1,2,3]},null,{"score":2,"v":[4,5,6]}]`)) + require.NoError(t, err) + defer values.Release() + col := arrow.NewColumnFromArr(field, values) + defer col.Release() + table := array.NewTable(arrow.NewSchema([]arrow.Field{field}, nil), []arrow.Column{col}, 3) + defer table.Release() + + var sink bytes.Buffer + require.NoError(t, pqarrow.WriteTable(table, &sink, 3, parquet.NewWriterProperties(parquet.WithAllocator(mem)), + pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithFiniteVectorLogicalType()))) + got, err := pqarrow.ReadTable(context.Background(), bytes.NewReader(sink.Bytes()), + parquet.NewReaderProperties(mem), pqarrow.ArrowReadProperties{}, mem) + require.NoError(t, err) + defer got.Release() + require.True(t, array.Equal(values, got.Column(0).Data().Chunk(0))) +} + +func TestWriteFiniteVectorRejectsDictionaryArray(t *testing.T) { + logical, err := schema.NewFiniteVectorLogicalType(2) + require.NoError(t, err) + element := schema.Must(schema.NewPrimitiveNodeLogical("element", parquet.Repetitions.Required, + nil, parquet.Types.Float, -1, -1)) + list := schema.Must(schema.NewGroupNode("list", parquet.Repetitions.Repeated, schema.FieldList{element}, -1)) + vector := schema.Must(schema.NewGroupNodeLogical("v", parquet.Repetitions.Required, schema.FieldList{list}, logical, -1)) + root := schema.MustGroup(schema.NewGroupNode("schema", parquet.Repetitions.Required, schema.FieldList{vector}, -1)) + + var sink bytes.Buffer + writer := file.NewParquetWriter(&sink, root, + file.WithWriterProps(parquet.NewWriterProperties(parquet.WithDictionaryDefault(false)))) + defer func() { require.NoError(t, writer.Close()) }() + group, err := writer.AppendRowGroupChecked() + require.NoError(t, err) + column, err := group.NextColumn() + require.NoError(t, err) + + mem := memory.DefaultAllocator + indexBuilder := array.NewInt32Builder(mem) + defer indexBuilder.Release() + indexBuilder.AppendValues([]int32{0, 1}, nil) + indices := indexBuilder.NewArray() + defer indices.Release() + valueBuilder := array.NewFloat32Builder(mem) + defer valueBuilder.Release() + valueBuilder.AppendValues([]float32{1, float32(math.NaN())}, nil) + values := valueBuilder.NewArray() + defer values.Release() + dictionary := array.NewDictionaryArray( + &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int32, ValueType: arrow.PrimitiveTypes.Float32}, + indices, values) + defer dictionary.Release() + + props := pqarrow.DefaultWriterProps() + ctx := pqarrow.NewArrowWriteContext(context.Background(), &props) + err = pqarrow.WriteArrowToColumn(ctx, column, dictionary, []int16{1, 1}, []int16{0, 1}, false) + require.ErrorIs(t, err, arrow.ErrInvalid) + require.ErrorContains(t, err, "dictionary arrays are not supported for FINITE_VECTOR elements") +} diff --git a/parquet/pqarrow/path_builder.go b/parquet/pqarrow/path_builder.go index 6f25bf506..13ed6e52d 100644 --- a/parquet/pqarrow/path_builder.go +++ b/parquet/pqarrow/path_builder.go @@ -310,6 +310,7 @@ type pathBuilder struct { info pathInfo paths []pathInfo nullableInParent bool + vectorLeaves []bool refCount *atomic.Int64 } @@ -465,15 +466,18 @@ func (p *pathBuilder) Visit(arr arrow.Array) error { } func (p *pathBuilder) addTerminalInfo(arr arrow.Array) { - p.info.leafIsNullable = p.nullableInParent - if p.nullableInParent { + // FINITE_VECTOR leaves are required on disk. Treat them as present when + // computing levels. The writer checks the visited values before writing. + vectorLeaf := len(p.paths) < len(p.vectorLeaves) && p.vectorLeaves[len(p.paths)] + p.info.leafIsNullable = p.nullableInParent && !vectorLeaf + if p.info.leafIsNullable { p.info.maxDefLevel++ } // we don't use null_count because if the null_count isn't known // and the array does in fact contain nulls, we will end up traversing // the null bitmap twice. - if lazyNoNulls(arr) { + if vectorLeaf || lazyNoNulls(arr) { p.info.path = append(p.info.path, &allPresentTerminalNode{p.info.maxDefLevel}) p.info.leafIsNullable = false } else if lazyNullCount(arr) == int64(arr.Len()) { @@ -530,11 +534,20 @@ func (m *multipathLevelBuilder) Release() { } func newMultipathLevelBuilder(arr arrow.Array, fieldNullable bool) (*multipathLevelBuilder, error) { + return buildMultipathLevels(arr, fieldNullable, nil) +} + +func buildMultipathLevels(arr arrow.Array, fieldNullable bool, vectorLeaves []bool) (*multipathLevelBuilder, error) { ret := &multipathLevelBuilder{ refCount: utils.NewRefCount(1), rootRange: elemRange{int64(0), int64(arr.Data().Len())}, data: arr.Data(), - builder: pathBuilder{nullableInParent: fieldNullable, paths: make([]pathInfo, 0), refCount: utils.NewRefCount(1)}, + builder: pathBuilder{ + nullableInParent: fieldNullable, + vectorLeaves: vectorLeaves, + paths: make([]pathInfo, 0), + refCount: utils.NewRefCount(1), + }, } if err := ret.builder.Visit(arr); err != nil { ret.builder.Release() diff --git a/parquet/pqarrow/properties.go b/parquet/pqarrow/properties.go index 0ead4cb98..deb584987 100644 --- a/parquet/pqarrow/properties.go +++ b/parquet/pqarrow/properties.go @@ -34,6 +34,7 @@ type ArrowWriterProperties struct { allowTruncatedTimestamps bool storeSchema bool noMapLogicalType bool + writeFiniteVector bool // compliantNestedTypes bool } @@ -113,6 +114,15 @@ func WithStoreSchema() WriterOption { } } +// WithFiniteVectorLogicalType writes eligible fixed-size lists as FINITE_VECTOR. +// This is off by default. A present vector cannot contain nulls, NaN or infinity. +// Readers must support FINITE_VECTOR to preserve the size without an Arrow schema. +func WithFiniteVectorLogicalType() WriterOption { + return func(c *config) { + c.props.writeFiniteVector = true + } +} + func WithNoMapLogicalType() WriterOption { return func(c *config) { c.props.noMapLogicalType = true diff --git a/parquet/pqarrow/schema.go b/parquet/pqarrow/schema.go index eeee4dbbe..cf8b859cf 100644 --- a/parquet/pqarrow/schema.go +++ b/parquet/pqarrow/schema.go @@ -320,6 +320,29 @@ func fieldToNode(name string, field arrow.Field, props *parquet.WriterProperties case arrow.FIXED_SIZE_LIST, arrow.LIST, arrow.LARGE_LIST: elemField := field.Type.(arrow.ListLikeType).ElemField() + if list, ok := field.Type.(*arrow.FixedSizeListType); ok && arrprops.writeFiniteVector && + list.Len() > 0 && isFiniteVectorValueType(elemField.Type) { + required := elemField + required.Nullable = false + element, err := fieldToNode("element", required, props, arrprops) + if err != nil { + return nil, err + } + if schema.IsFiniteVectorElement(element) { + logical, err := schema.NewFiniteVectorLogicalType(list.Len()) + if err != nil { + return nil, err + } + middle, err := schema.NewGroupNode("list", parquet.Repetitions.Repeated, + schema.FieldList{element}, -1) + if err != nil { + return nil, err + } + return schema.NewGroupNodeLogical(name, repType, schema.FieldList{middle}, + logical, fieldIDFromMeta(field.Metadata)) + } + } + child, err := fieldToNode(name, elemField, props, arrprops) if err != nil { return nil, err @@ -374,6 +397,19 @@ func fieldToNode(name string, field arrow.Field, props *parquet.WriterProperties return schema.NewPrimitiveNodeLogical(name, repType, logicalType, typ, length, fieldIDFromMeta(field.Metadata)) } +func isFiniteVectorValueType(typ arrow.DataType) bool { + for typ.ID() == arrow.EXTENSION { + typ = typ.(arrow.ExtensionType).StorageType() + } + switch typ.ID() { + case arrow.BOOL, arrow.INT8, arrow.INT16, arrow.INT32, arrow.INT64, + arrow.UINT8, arrow.UINT16, arrow.UINT32, arrow.UINT64, + arrow.FLOAT16, arrow.FLOAT32, arrow.FLOAT64, arrow.DECIMAL, arrow.DECIMAL256: + return true + } + return false +} + const fieldIDKey = "PARQUET:field_id" func fieldIDFromMeta(m arrow.Metadata) int32 { @@ -978,7 +1014,13 @@ func variantToSchemaField(n *schema.GroupNode, currentLevels file.LevelInfo, ctx } func groupToSchemaField(n *schema.GroupNode, currentLevels file.LevelInfo, ctx *schemaTree, parent, out *SchemaField) error { - if n.LogicalType().Equals(schema.NewListLogicalType()) { + if vector, ok := n.LogicalType().(schema.FiniteVectorLogicalType); ok { + if err := listToSchemaField(n, currentLevels, ctx, parent, out); err != nil { + return err + } + out.Field.Type = arrow.FixedSizeListOfField(vector.NumElements(), *out.Children[0].Field) + return nil + } else if n.LogicalType().Equals(schema.NewListLogicalType()) { return listToSchemaField(n, currentLevels, ctx, parent, out) } else if n.LogicalType().Equals(schema.MapLogicalType{}) { return mapToSchemaField(n, currentLevels, ctx, parent, out) @@ -1131,6 +1173,13 @@ func getNestedFactory(origin, inferred arrow.DataType) func(fieldList []arrow.Fi return arrow.LargeListOfField(list[0]) } } + case arrow.FIXED_SIZE_LIST: + size := inferred.(*arrow.FixedSizeListType).Len() + if original, ok := origin.(*arrow.FixedSizeListType); ok && original.Len() == size { + return func(list []arrow.Field) arrow.DataType { + return arrow.FixedSizeListOfField(size, list[0]) + } + } case arrow.MAP: if origin.ID() == arrow.MAP { return func(list []arrow.Field) arrow.DataType { @@ -1209,11 +1258,17 @@ func applyOriginalStorageMetadata(origin arrow.Field, inferred *SchemaField) (mo } modified = origin.Type.ID() != inferred.Field.Type.ID() - childModified, err := applyOriginalMetadata(arrow.Field{Type: origin.Type.(arrow.ListLikeType).Elem()}, &inferred.Children[0]) + originalElement := origin.Type.(arrow.ListLikeType).ElemField() + childModified, err := applyOriginalMetadata(originalElement, &inferred.Children[0]) if err != nil { return modified, err } modified = modified || childModified + if inferred.Field.Type.ID() == arrow.FIXED_SIZE_LIST && + originalElement.Nullable && !inferred.Children[0].Field.Nullable { + inferred.Children[0].Field.Nullable = true + modified = true + } if modified { inferred.Field.Type = factory([]arrow.Field{*inferred.Children[0].Field}) } diff --git a/parquet/schema/finite_vector_test.go b/parquet/schema/finite_vector_test.go new file mode 100644 index 000000000..238a415ea --- /dev/null +++ b/parquet/schema/finite_vector_test.go @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "testing" + + "github.com/apache/arrow-go/v18/parquet" + "github.com/stretchr/testify/require" +) + +func TestFiniteVectorSchema(t *testing.T) { + logical, err := NewFiniteVectorLogicalType(3) + require.NoError(t, err) + _, err = NewFiniteVectorLogicalType(0) + require.Error(t, err) + + element, err := NewPrimitiveNodeLogical("element", parquet.Repetitions.Required, + nil, parquet.Types.Float, -1, -1) + require.NoError(t, err) + list, err := NewGroupNode("list", parquet.Repetitions.Repeated, FieldList{element}, -1) + require.NoError(t, err) + + for _, annotation := range []LogicalType{logical, &logical} { + _, err = NewGroupNodeLogical("v", parquet.Repetitions.Repeated, FieldList{list}, annotation, -1) + require.Error(t, err) + require.Nil(t, list.Parent(), "a rejected group must not become the child's parent") + } + _, err = NewGroupNodeLogical("v", parquet.Repetitions.Optional, FieldList{list}, (*FiniteVectorLogicalType)(nil), -1) + require.Error(t, err) + + vector, err := NewGroupNodeLogical("v", parquet.Repetitions.Optional, FieldList{list}, &logical, -1) + require.NoError(t, err) + require.Equal(t, logical, vector.LogicalType()) + root, err := NewGroupNode("schema", parquet.Repetitions.Required, FieldList{vector}, -1) + require.NoError(t, err) + + encoded := ToThrift(root) + require.Equal(t, int32(3), encoded[1].LogicalType.FINITE_VECTOR.NumElements) + require.Equal(t, ConvertedTypes.List, ConvertedType(encoded[1].GetConvertedType())) + decoded, err := FromParquet(encoded) + require.NoError(t, err) + require.True(t, root.Equals(decoded)) + + encoded[1].LogicalType.FINITE_VECTOR.NumElements = 0 + _, err = FromParquet(encoded) + require.ErrorContains(t, err, "must have a positive size") +} diff --git a/parquet/schema/logical_types.go b/parquet/schema/logical_types.go index f1e85bd0a..ad97d8033 100644 --- a/parquet/schema/logical_types.go +++ b/parquet/schema/logical_types.go @@ -43,6 +43,8 @@ func getLogicalType(l *format.LogicalType) LogicalType { return MapLogicalType{} case l.IsSetLIST(): return ListLogicalType{} + case l.IsSetFINITE_VECTOR(): + return FiniteVectorLogicalType{numElements: l.FINITE_VECTOR.NumElements} case l.IsSetENUM(): return EnumLogicalType{} case l.IsSetDECIMAL(): @@ -304,6 +306,55 @@ func (ListLogicalType) Equals(rhs LogicalType) bool { return ok } +// FiniteVectorLogicalType describes an ordinary Parquet list with a fixed +// number of required, finite elements. +type FiniteVectorLogicalType struct { + baseLogicalType + numElements int32 +} + +// NewFiniteVectorLogicalType creates a vector annotation with a positive size. +func NewFiniteVectorLogicalType(numElements int32) (FiniteVectorLogicalType, error) { + if numElements <= 0 { + return FiniteVectorLogicalType{}, fmt.Errorf("FINITE_VECTOR requires a positive number of elements, got %d", numElements) + } + return FiniteVectorLogicalType{numElements: numElements}, nil +} + +// NumElements returns the number of elements in each present vector. +func (t FiniteVectorLogicalType) NumElements() int32 { return t.numElements } + +func (FiniteVectorLogicalType) SortOrder() SortOrder { return SortUNKNOWN } +func (FiniteVectorLogicalType) IsNested() bool { return true } +func (FiniteVectorLogicalType) IsApplicable(parquet.Type, int32) bool { + return false +} + +func (FiniteVectorLogicalType) ToConvertedType() (ConvertedType, DecimalMetadata) { + return ConvertedTypes.List, DecimalMetadata{} +} + +func (FiniteVectorLogicalType) IsCompatible(t ConvertedType, dec DecimalMetadata) bool { + return t == ConvertedTypes.List && !dec.IsSet +} + +func (t FiniteVectorLogicalType) String() string { + return fmt.Sprintf("FiniteVector(num_elements=%d)", t.numElements) +} + +func (t FiniteVectorLogicalType) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]any{"Type": "FiniteVector", "num_elements": t.numElements}) +} + +func (t FiniteVectorLogicalType) toThrift() *format.LogicalType { + return &format.LogicalType{FINITE_VECTOR: &format.FiniteVectorType{NumElements: t.numElements}} +} + +func (t FiniteVectorLogicalType) Equals(rhs LogicalType) bool { + other, ok := rhs.(FiniteVectorLogicalType) + return ok && t.numElements == other.numElements +} + // EnumLogicalType is for representing an enum, which should be a byte array type type EnumLogicalType struct{ baseLogicalType } diff --git a/parquet/schema/node.go b/parquet/schema/node.go index 75ae1c5e4..c7242d444 100644 --- a/parquet/schema/node.go +++ b/parquet/schema/node.go @@ -407,6 +407,14 @@ func NewGroupNodeConverted(name string, repetition parquet.Repetition, fields Fi // NewGroupNodeLogical constructs a group node with the provided fields and logical type, // determining the converted type from the provided logical type. func NewGroupNodeLogical(name string, repetition parquet.Repetition, fields FieldList, logical LogicalType, id int32) (n *GroupNode, err error) { + // Use the value form throughout the schema so validation and type mapping + // handle vectors the same way regardless of how the caller passes them. + if vector, ok := logical.(*FiniteVectorLogicalType); ok { + if vector == nil { + return nil, fmt.Errorf("FINITE_VECTOR %q is nil", name) + } + logical = *vector + } n = &GroupNode{ node: node{typ: Group, name: name, repetition: repetition, logicalType: logical, fieldID: id}, fields: fields, @@ -429,6 +437,12 @@ func NewGroupNodeLogical(name string, repetition parquet.Repetition, fields Fiel return } + if vector, ok := logical.(FiniteVectorLogicalType); ok { + if err := validateFiniteVector(n, vector); err != nil { + return nil, err + } + } + n.nameToIdx = make(strIntMultimap) for idx, f := range n.fields { f.SetParent(n) @@ -437,6 +451,51 @@ func NewGroupNodeLogical(name string, repetition parquet.Repetition, fields Fiel return } +func validateFiniteVector(n *GroupNode, logical FiniteVectorLogicalType) error { + if logical.NumElements() <= 0 { + return fmt.Errorf("FINITE_VECTOR %q must have a positive size", n.Name()) + } + if (n.RepetitionType() != parquet.Repetitions.Required && n.RepetitionType() != parquet.Repetitions.Optional) || n.NumFields() != 1 { + return fmt.Errorf("FINITE_VECTOR %q must be a required or optional group with one child", n.Name()) + } + list, ok := n.Field(0).(*GroupNode) + if !ok || list.Name() != "list" || list.RepetitionType() != parquet.Repetitions.Repeated || + !list.LogicalType().IsNone() || list.NumFields() != 1 { + return fmt.Errorf("FINITE_VECTOR %q must contain one unannotated repeated group named list", n.Name()) + } + element := list.Field(0) + if element.Name() != "element" || !IsFiniteVectorElement(element) { + return fmt.Errorf("FINITE_VECTOR %q must contain a supported required primitive named element", n.Name()) + } + return nil +} + +// IsFiniteVectorElement reports whether the node can be the required primitive +// element of a FINITE_VECTOR. +func IsFiniteVectorElement(n Node) bool { + p, ok := n.(*PrimitiveNode) + if !ok || p.RepetitionType() != parquet.Repetitions.Required { + return false + } + logical := p.LogicalType() + switch p.PhysicalType() { + case parquet.Types.Boolean, parquet.Types.Float, parquet.Types.Double: + return logical.IsNone() + case parquet.Types.Int32, parquet.Types.Int64: + switch logical.(type) { + case IntLogicalType, DecimalLogicalType: + return true + } + return logical.IsNone() + case parquet.Types.FixedLenByteArray: + switch logical.(type) { + case Float16LogicalType, DecimalLogicalType: + return true + } + } + return false +} + // NewGroupNode constructs a new group node with the provided fields, // but with converted type None and No Logical Type func NewGroupNode(name string, repetition parquet.Repetition, fields FieldList, fieldID int32) (*GroupNode, error) {