Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 53 additions & 70 deletions table/internal/variant_shredding.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package internal

import (
"slices"
"sort"

"github.com/apache/arrow-go/v18/arrow"
Expand All @@ -27,10 +26,11 @@ import (
"github.com/apache/arrow-go/v18/parquet/variant"
)

// Variant shredding inference: most-common-type selection with explicit
// tie-break, integer/decimal widening, and frequency/depth/field-count bounds.
// Field caps are per-object-node (matching Java); no global budget - total is
// bounded by the sample, maxShreddingDepth, and the per-node caps.
// Variant shredding inference: a field shreds only when all its non-null values
// share one type family (the integer and decimal families widen to their widest
// observed member); a field mixing more than one family is not shreddable and
// stays in the residual value. Frequency/depth/field-count bounds and per-object-
// node caps match Java; no global budget.
const (
minFieldFrequency = 0.10
maxShreddedFields = 300
Expand All @@ -47,35 +47,26 @@ var decimalPriority = map[variant.Type]int{
variant.Decimal4: 0, variant.Decimal8: 1, variant.Decimal16: 2,
}

// tieBreakPriority breaks count ties in most-common-type selection; higher wins.
// Types absent here (Null, Array, Object) resolve to -1.
var tieBreakPriority = map[variant.Type]int{
variant.Bool: 0, variant.Int8: 1, variant.Int16: 2, variant.Int32: 3,
variant.Int64: 4, variant.Float: 5, variant.Double: 6, variant.Decimal4: 7,
variant.Decimal8: 8, variant.Decimal16: 9, variant.Date: 10, variant.Time: 11,
variant.TimestampMicros: 12, variant.TimestampMicrosNTZ: 13, variant.Binary: 14,
variant.String: 15, variant.TimestampNanos: 16, variant.TimestampNanosNTZ: 17,
variant.UUID: 18,
}

type fieldInfo struct {
typeCounts map[variant.Type]int
admitted variant.Type
hasAdmitted bool
mixed bool
observationCount int
maxDecimalScale int
maxDecimalIntDigits int
}

func newFieldInfo() *fieldInfo {
return &fieldInfo{typeCounts: make(map[variant.Type]int)}
return &fieldInfo{}
}

// observe records one value's type at this node (per-value counting).
// observe records one value at this node, folding its type into the admitted type.
func (f *fieldInfo) observe(v variant.Value) {
f.observationCount++

t := v.Type()
// arrow-go has a single Bool type, so no false-folds-to-true step.
f.typeCounts[t]++
f.admit(t)

if isDecimalType(t) {
intDigits, scale := decimalDigits(v.Value())
Expand All @@ -88,67 +79,59 @@ func (f *fieldInfo) observe(v variant.Value) {
}
}

// mostCommonType collapses int/decimal families to the widest, then picks max by count.
func (f *fieldInfo) mostCommonType() (variant.Type, bool) {
combined := make(map[variant.Type]int)

intTotal, decTotal := 0, 0
var widestInt, widestDec variant.Type
haveInt, haveDec := false, false

for t, c := range f.typeCounts {
switch {
case isIntegerType(t):
intTotal += c
if !haveInt || integerPriority[t] > integerPriority[widestInt] {
widestInt, haveInt = t, true
}
case isDecimalType(t):
decTotal += c
if !haveDec || decimalPriority[t] > decimalPriority[widestDec] {
widestDec, haveDec = t, true
}
default:
combined[t] = c
}
// admit widens the admitted type with t, or marks the node mixed when t belongs to
// a different family.
func (f *fieldInfo) admit(t variant.Type) {
if f.mixed {
return
}
if haveInt {
combined[widestInt] = intTotal
if !f.hasAdmitted {
f.admitted, f.hasAdmitted = t, true

return
}
if haveDec {
combined[widestDec] = decTotal
merged, ok := mergeFamily(f.admitted, t)
if !ok {
f.mixed = true

return
}
f.admitted = merged
}

if len(combined) == 0 {
return 0, false
// mergeFamily widens current with candidate within the integer or decimal family,
// returning ok=false when they belong to different families (mirroring Java's
// VariantShreddingAnalyzer.mergeFamily).
func mergeFamily(current, candidate variant.Type) (variant.Type, bool) {
if current == candidate {
return current, true
}
if isIntegerType(current) && isIntegerType(candidate) {
if integerPriority[candidate] > integerPriority[current] {
return candidate, true
}

// Max by count, then tieBreakPriority, then type value (stable, sorted keys).
var best variant.Type
bestCount, bestPrio := -1, -2
first := true
keys := make([]variant.Type, 0, len(combined))
for t := range combined {
keys = append(keys, t)
return current, true
}
slices.Sort(keys)
for _, t := range keys {
c := combined[t]
p := tiePriority(t)
if first || c > bestCount || (c == bestCount && p > bestPrio) {
best, bestCount, bestPrio, first = t, c, p, false
if isDecimalType(current) && isDecimalType(candidate) {
if decimalPriority[candidate] > decimalPriority[current] {
return candidate, true
}

return current, true
}

return best, true
return 0, false
}

func tiePriority(t variant.Type) int {
if p, ok := tieBreakPriority[t]; ok {
return p
// admittedType returns the single type this node shreds to, or ok=false when the
// node saw no values or mixed more than one type family.
func (f *fieldInfo) admittedType() (variant.Type, bool) {
if !f.hasAdmitted || f.mixed {
return 0, false
}

return -1
return f.admitted, true
}

func isIntegerType(t variant.Type) bool {
Expand Down Expand Up @@ -213,7 +196,7 @@ func AnalyzeVariantShredding(sample []variant.Value) (arrow.DataType, bool) {
traverseVariant(root, v, 0)
}

rootType, ok := root.info.mostCommonType()
rootType, ok := root.info.admittedType()
if !ok {
return nil, false
}
Expand Down Expand Up @@ -324,7 +307,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType {
fields := make([]arrow.Field, 0, len(names))
for _, name := range names {
child := node.objectChildren[name]
ct, ok := child.info.mostCommonType()
ct, ok := child.info.admittedType()
if !ok {
continue
}
Expand All @@ -343,7 +326,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType {
if node.arrayElement == nil {
return nil
}
et, ok := node.arrayElement.info.mostCommonType()
et, ok := node.arrayElement.info.admittedType()
if !ok {
return nil
}
Expand Down
105 changes: 88 additions & 17 deletions table/internal/variant_shredding_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,18 @@ func mkVars(t *testing.T, raws ...any) []variant.Value {
return out
}

// mkBuilt builds a variant.Value via a builder callback, for leaf types (decimal,
// timestamp) that the reflection-based mkVar cannot express directly.
func mkBuilt(t *testing.T, build func(*variant.Builder) error) variant.Value {
t.Helper()
var b variant.Builder
require.NoError(t, build(&b))
v, err := b.Build()
require.NoError(t, err)

return v
}

// bigI64 is large enough to force Int64 encoding (> 2^32).
const bigI64 = int64(5_000_000_000)

Expand Down Expand Up @@ -115,22 +127,89 @@ func TestAnalyzeIntegerWidening(t *testing.T) {
"widening must pick Int64, got %s", f[0].Type)
}

func TestAnalyzeMixedTypeMajority(t *testing.T) {
// "v" is an int in 7 rows, a string in 3 -> majority int wins.
// TestAnalyzeMixedTypeNotShredded: a field mixing type families (int in 7 rows,
// string in 3) is not uniform and must not shred - it is this object's only
// field, so nothing shreds at all.
func TestAnalyzeMixedTypeNotShredded(t *testing.T) {
var sample []variant.Value
for range 7 {
sample = append(sample, mkVar(t, map[string]any{"v": bigI64}))
}
for range 3 {
sample = append(sample, mkVar(t, map[string]any{"v": "s"}))
}
_, ok := AnalyzeVariantShredding(sample)
assert.False(t, ok, "mixed int+string field must not shred")
}

// TestAnalyzeCrossFamilyNotShredded: distinct scalar families that do not widen
// into each other are not uniform and must not shred.
func TestAnalyzeCrossFamilyNotShredded(t *testing.T) {
microsTS := func(b *variant.Builder) error {
return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), true, true)
}
nanosTS := func(b *variant.Builder) error {
return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), false, true)
}
dec := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(12345)) }

for _, c := range []struct {
name string
sample []variant.Value
}{
{"float+double", mkVars(t, float32(1.5), float64(2.5))},
{"timestamp micros+nanos", []variant.Value{mkBuilt(t, microsTS), mkBuilt(t, nanosTS)}},
{"int+decimal", []variant.Value{mkVar(t, bigI64), mkBuilt(t, dec)}},
} {
t.Run(c.name, func(t *testing.T) {
_, ok := AnalyzeVariantShredding(c.sample)
assert.Falsef(t, ok, "%s must not shred", c.name)
})
}
}

// TestAnalyzeMixedSiblingKeepsUniform: a mixed-type field is dropped from the
// shredded type while its uniform sibling still shreds.
func TestAnalyzeMixedSiblingKeepsUniform(t *testing.T) {
sample := mkVars(t,
map[string]any{"good": bigI64, "bad": bigI64},
map[string]any{"good": bigI64, "bad": "s"},
)
dt, ok := AnalyzeVariantShredding(sample)
require.True(t, ok)
st := dt.(*arrow.StructType)
f, _ := st.FieldsByName("v")
require.Len(t, f, 1)
assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, f[0].Type),
"majority int must win, got %s", f[0].Type)

want := arrow.StructOf(arrow.Field{Name: "good", Type: arrow.PrimitiveTypes.Int64, Nullable: true})
assert.Truef(t, arrow.TypeEqual(want, dt), "want %s got %s", want, dt)
}

// TestAnalyzeEmptyContainerNotShredded: an object with no fields and an array with no
// elements have nothing to shred, so inference must decline.
func TestAnalyzeEmptyContainerNotShredded(t *testing.T) {
_, ok := AnalyzeVariantShredding(mkVars(t, map[string]any{}, map[string]any{}))
assert.False(t, ok, "empty object must not shred")

_, ok = AnalyzeVariantShredding(mkVars(t, []any{}, []any{}))
assert.False(t, ok, "empty array must not shred")
}

// TestAnalyzeDecimalWidening: the decimal family widens to a single leaf and still
// shreds; only cross-family mixing blocks shredding.
func TestAnalyzeDecimalWidening(t *testing.T) {
dec4 := func(b *variant.Builder) error { return b.AppendDecimal4(2, decimal.Decimal32(12345)) }
dec8 := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(1234567890)) }
dt, ok := AnalyzeVariantShredding([]variant.Value{mkBuilt(t, dec4), mkBuilt(t, dec8)})
require.True(t, ok, "decimal family must still shred after widening")
_, isDec := dt.(*arrow.Decimal128Type)
assert.Truef(t, isDec, "decimal widening must produce a decimal leaf, got %s", dt)
}

// TestAnalyzeNullsPlusOneType: variant-null values are not observed, so a field
// null in some rows and one type in the rest still shreds as that type.
func TestAnalyzeNullsPlusOneType(t *testing.T) {
dt, ok := AnalyzeVariantShredding(mkVars(t, nil, bigI64, nil, bigI64))
require.True(t, ok)
assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, dt),
"nulls + int must shred as Int64, got %s", dt)
}

func TestAnalyzeScalarRoot(t *testing.T) {
Expand Down Expand Up @@ -268,22 +347,14 @@ func TestInferredTypedScalarsShred(t *testing.T) {
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
mk := func() variant.Value {
var b variant.Builder
require.NoError(t, c.build(&b))
v, err := b.Build()
require.NoError(t, err)

return v
}
v := mk()
v := mkBuilt(t, c.build)
inner, ok := AnalyzeVariantShredding([]variant.Value{v})
require.Truef(t, ok, "%s should infer a shredded type", c.name)

st := extensions.NewShreddedVariantType(inner)
bldr := extensions.NewVariantBuilder(mem, st)
defer bldr.Release()
bldr.Append(mk())
bldr.Append(v)
arr := bldr.NewArray().(*extensions.VariantArray)
defer arr.Release()

Expand Down
13 changes: 7 additions & 6 deletions table/variant_shredded_write_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -578,8 +578,8 @@ func TestShreddedVariantWriteScalarTypes(t *testing.T) {
}

// TestShreddedVariantWriteMixedTypeField writes an object field that is int64 in most
// rows and string in a few. Inference shreds it as int64 (the majority); the minority
// string rows must round-trip through the unshredded residual, not be dropped.
// rows and string in a few. The mixed field is not uniform, so it is not shredded at
// all; every row must still round-trip through the unshredded value.
func TestShreddedVariantWriteMixedTypeField(t *testing.T) {
mem := memory.DefaultAllocator
iceSchema := iceberg.NewSchema(0,
Expand Down Expand Up @@ -633,7 +633,8 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) {
files = append(files, df)
}
require.Len(t, files, 1)
assert.True(t, payloadHasTypedValue(t, files[0].FilePath()), "f shreds as int64 (majority)")
assert.False(t, payloadHasTypedValue(t, files[0].FilePath()),
"mixed-type field is not uniform, so payload must not shred")

p := strings.TrimPrefix(files[0].FilePath(), "file://")
f, err := os.Open(p)
Expand All @@ -656,10 +657,10 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) {
assert.JSONEqf(t, want, string(got), "row %d (mixed-type minority must round-trip)", i)
}

// f shreds as int64 but the string rows land in the residual value column, so its
// typed stats do not cover all values: the bound must be dropped (Java value() rule).
// The mixed-type field is not shredded, so there is no typed_value to derive stats
// from: the field must not emit a variant bound.
_, hasLower := files[0].LowerBoundValues()[2]
assert.False(t, hasLower, "partial-shred field must not emit a variant bound")
assert.False(t, hasLower, "unshredded field must not emit a variant bound")
}

// TestShreddedVariantWriteRowConservation verifies every row survives bootstrap ->
Expand Down
Loading