From 42121fa368e4366a29b0d5c1320fe53fef47b07a Mon Sep 17 00:00:00 2001 From: Neelesh Salian Date: Tue, 18 Aug 2026 13:56:33 -0700 Subject: [PATCH 1/2] feat(table): type uniformity for variant shredding inference Signed-off-by: Neelesh Salian --- table/internal/variant_shredding.go | 94 +++++++------------ table/internal/variant_shredding_test.go | 110 +++++++++++++++++++---- table/variant_shredded_write_test.go | 13 +-- 3 files changed, 135 insertions(+), 82 deletions(-) diff --git a/table/internal/variant_shredding.go b/table/internal/variant_shredding.go index dd0c90d5f0..847014f2f4 100644 --- a/table/internal/variant_shredding.go +++ b/table/internal/variant_shredding.go @@ -18,7 +18,6 @@ package internal import ( - "slices" "sort" "github.com/apache/arrow-go/v18/arrow" @@ -27,10 +26,11 @@ import ( "github.com/apache/arrow-go/v18/parquet/variant" ) -// Variant shredding inference: most-common-type selection with explicit -// tie-break, integer/decimal widening, and frequency/depth/field-count bounds. -// Field caps are per-object-node (matching Java); no global budget - total is -// bounded by the sample, maxShreddingDepth, and the per-node caps. +// Variant shredding inference: a field shreds only when all its non-null values +// share one type family (the integer and decimal families widen to their widest +// observed member); a field mixing more than one family is not shreddable and +// stays in the residual value. Frequency/depth/field-count bounds and per-object- +// node caps match Java; no global budget. const ( minFieldFrequency = 0.10 maxShreddedFields = 300 @@ -47,17 +47,6 @@ var decimalPriority = map[variant.Type]int{ variant.Decimal4: 0, variant.Decimal8: 1, variant.Decimal16: 2, } -// tieBreakPriority breaks count ties in most-common-type selection; higher wins. -// Types absent here (Null, Array, Object) resolve to -1. -var tieBreakPriority = map[variant.Type]int{ - variant.Bool: 0, variant.Int8: 1, variant.Int16: 2, variant.Int32: 3, - variant.Int64: 4, variant.Float: 5, variant.Double: 6, variant.Decimal4: 7, - variant.Decimal8: 8, variant.Decimal16: 9, variant.Date: 10, variant.Time: 11, - variant.TimestampMicros: 12, variant.TimestampMicrosNTZ: 13, variant.Binary: 14, - variant.String: 15, variant.TimestampNanos: 16, variant.TimestampNanosNTZ: 17, - variant.UUID: 18, -} - type fieldInfo struct { typeCounts map[variant.Type]int observationCount int @@ -88,67 +77,52 @@ func (f *fieldInfo) observe(v variant.Value) { } } -// mostCommonType collapses int/decimal families to the widest, then picks max by count. -func (f *fieldInfo) mostCommonType() (variant.Type, bool) { - combined := make(map[variant.Type]int) - - intTotal, decTotal := 0, 0 - var widestInt, widestDec variant.Type +// admittedType returns the single type this node shreds to. The integer and +// decimal families each collapse to their widest observed member; if more than one +// family remains the field is mixed-type and not shreddable (ok=false), mirroring +// Java VariantShreddingAnalyzer's admittedType. +func (f *fieldInfo) admittedType() (variant.Type, bool) { + var widestInt, widestDec, other variant.Type haveInt, haveDec := false, false + otherFamilies := 0 - for t, c := range f.typeCounts { + for t := range f.typeCounts { switch { case isIntegerType(t): - intTotal += c if !haveInt || integerPriority[t] > integerPriority[widestInt] { - widestInt, haveInt = t, true + widestInt = t } + haveInt = true case isDecimalType(t): - decTotal += c if !haveDec || decimalPriority[t] > decimalPriority[widestDec] { - widestDec, haveDec = t, true + widestDec = t } + haveDec = true default: - combined[t] = c + other = t + otherFamilies++ } } + + families := otherFamilies if haveInt { - combined[widestInt] = intTotal + families++ } if haveDec { - combined[widestDec] = decTotal + families++ } - - if len(combined) == 0 { + if families != 1 { return 0, false } - // Max by count, then tieBreakPriority, then type value (stable, sorted keys). - var best variant.Type - bestCount, bestPrio := -1, -2 - first := true - keys := make([]variant.Type, 0, len(combined)) - for t := range combined { - keys = append(keys, t) - } - slices.Sort(keys) - for _, t := range keys { - c := combined[t] - p := tiePriority(t) - if first || c > bestCount || (c == bestCount && p > bestPrio) { - best, bestCount, bestPrio, first = t, c, p, false - } - } - - return best, true -} - -func tiePriority(t variant.Type) int { - if p, ok := tieBreakPriority[t]; ok { - return p + switch { + case haveInt: + return widestInt, true + case haveDec: + return widestDec, true + default: + return other, true } - - return -1 } func isIntegerType(t variant.Type) bool { @@ -213,7 +187,7 @@ func AnalyzeVariantShredding(sample []variant.Value) (arrow.DataType, bool) { traverseVariant(root, v, 0) } - rootType, ok := root.info.mostCommonType() + rootType, ok := root.info.admittedType() if !ok { return nil, false } @@ -324,7 +298,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType { fields := make([]arrow.Field, 0, len(names)) for _, name := range names { child := node.objectChildren[name] - ct, ok := child.info.mostCommonType() + ct, ok := child.info.admittedType() if !ok { continue } @@ -343,7 +317,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType { if node.arrayElement == nil { return nil } - et, ok := node.arrayElement.info.mostCommonType() + et, ok := node.arrayElement.info.admittedType() if !ok { return nil } diff --git a/table/internal/variant_shredding_test.go b/table/internal/variant_shredding_test.go index d1754b88cc..a0300131c1 100644 --- a/table/internal/variant_shredding_test.go +++ b/table/internal/variant_shredding_test.go @@ -53,6 +53,18 @@ func mkVars(t *testing.T, raws ...any) []variant.Value { return out } +// mkBuilt builds a variant.Value via a builder callback, for leaf types (decimal, +// timestamp) that the reflection-based mkVar cannot express directly. +func mkBuilt(t *testing.T, build func(*variant.Builder) error) variant.Value { + t.Helper() + var b variant.Builder + require.NoError(t, build(&b)) + v, err := b.Build() + require.NoError(t, err) + + return v +} + // bigI64 is large enough to force Int64 encoding (> 2^32). const bigI64 = int64(5_000_000_000) @@ -115,8 +127,10 @@ func TestAnalyzeIntegerWidening(t *testing.T) { "widening must pick Int64, got %s", f[0].Type) } -func TestAnalyzeMixedTypeMajority(t *testing.T) { - // "v" is an int in 7 rows, a string in 3 -> majority int wins. +// TestAnalyzeMixedTypeNotShredded: a field mixing type families (int in 7 rows, +// string in 3) is not uniform and must not shred - it is this object's only +// field, so nothing shreds at all. +func TestAnalyzeMixedTypeNotShredded(t *testing.T) { var sample []variant.Value for range 7 { sample = append(sample, mkVar(t, map[string]any{"v": bigI64})) @@ -124,13 +138,85 @@ func TestAnalyzeMixedTypeMajority(t *testing.T) { for range 3 { sample = append(sample, mkVar(t, map[string]any{"v": "s"})) } + _, ok := AnalyzeVariantShredding(sample) + assert.False(t, ok, "mixed int+string field must not shred") +} + +// TestAnalyzeCrossFamilyNotShredded: distinct scalar families that do not widen +// into each other are not uniform and must not shred. +func TestAnalyzeCrossFamilyNotShredded(t *testing.T) { + microsTS := func(b *variant.Builder) error { + return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), true, true) + } + nanosTS := func(b *variant.Builder) error { + return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), false, true) + } + dec := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(12345)) } + + for _, c := range []struct { + name string + sample []variant.Value + }{ + {"float+double", mkVars(t, float32(1.5), float64(2.5))}, + {"timestamp micros+nanos", []variant.Value{mkBuilt(t, microsTS), mkBuilt(t, nanosTS)}}, + {"int+decimal", []variant.Value{mkVar(t, bigI64), mkBuilt(t, dec)}}, + } { + t.Run(c.name, func(t *testing.T) { + _, ok := AnalyzeVariantShredding(c.sample) + assert.Falsef(t, ok, "%s must not shred", c.name) + }) + } +} + +// TestAnalyzeMixedSiblingKeepsUniform: a mixed-type field is dropped from the +// shredded type while its uniform sibling still shreds. +func TestAnalyzeMixedSiblingKeepsUniform(t *testing.T) { + var sample []variant.Value + for range 8 { + sample = append(sample, mkVar(t, map[string]any{"good": bigI64, "bad": bigI64})) + } + for range 2 { + sample = append(sample, mkVar(t, map[string]any{"good": bigI64, "bad": "s"})) + } dt, ok := AnalyzeVariantShredding(sample) require.True(t, ok) st := dt.(*arrow.StructType) - f, _ := st.FieldsByName("v") - require.Len(t, f, 1) - assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, f[0].Type), - "majority int must win, got %s", f[0].Type) + + g, hasGood := st.FieldsByName("good") + require.True(t, hasGood, "uniform sibling must shred") + assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, g[0].Type), "good shreds as Int64, got %s", g[0].Type) + _, hasBad := st.FieldsByName("bad") + assert.False(t, hasBad, "mixed sibling must be dropped from the shredded type") +} + +// TestAnalyzeEmptyContainerNotShredded: an object with no fields and an array with no +// elements have nothing to shred, so inference must decline. +func TestAnalyzeEmptyContainerNotShredded(t *testing.T) { + _, ok := AnalyzeVariantShredding(mkVars(t, map[string]any{}, map[string]any{})) + assert.False(t, ok, "empty object must not shred") + + _, ok = AnalyzeVariantShredding(mkVars(t, []any{}, []any{})) + assert.False(t, ok, "empty array must not shred") +} + +// TestAnalyzeDecimalWidening: the decimal family widens to a single leaf and still +// shreds; only cross-family mixing blocks shredding. +func TestAnalyzeDecimalWidening(t *testing.T) { + dec4 := func(b *variant.Builder) error { return b.AppendDecimal4(2, decimal.Decimal32(12345)) } + dec8 := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(1234567890)) } + dt, ok := AnalyzeVariantShredding([]variant.Value{mkBuilt(t, dec4), mkBuilt(t, dec8)}) + require.True(t, ok, "decimal family must still shred after widening") + _, isDec := dt.(*arrow.Decimal128Type) + assert.Truef(t, isDec, "decimal widening must produce a decimal leaf, got %s", dt) +} + +// TestAnalyzeNullsPlusOneType: variant-null values are not observed, so a field +// null in some rows and one type in the rest still shreds as that type. +func TestAnalyzeNullsPlusOneType(t *testing.T) { + dt, ok := AnalyzeVariantShredding(mkVars(t, nil, bigI64, nil, bigI64)) + require.True(t, ok) + assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, dt), + "nulls + int must shred as Int64, got %s", dt) } func TestAnalyzeScalarRoot(t *testing.T) { @@ -268,22 +354,14 @@ func TestInferredTypedScalarsShred(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - mk := func() variant.Value { - var b variant.Builder - require.NoError(t, c.build(&b)) - v, err := b.Build() - require.NoError(t, err) - - return v - } - v := mk() + v := mkBuilt(t, c.build) inner, ok := AnalyzeVariantShredding([]variant.Value{v}) require.Truef(t, ok, "%s should infer a shredded type", c.name) st := extensions.NewShreddedVariantType(inner) bldr := extensions.NewVariantBuilder(mem, st) defer bldr.Release() - bldr.Append(mk()) + bldr.Append(mkBuilt(t, c.build)) arr := bldr.NewArray().(*extensions.VariantArray) defer arr.Release() diff --git a/table/variant_shredded_write_test.go b/table/variant_shredded_write_test.go index 9e4b8ec92e..7c8f639e36 100644 --- a/table/variant_shredded_write_test.go +++ b/table/variant_shredded_write_test.go @@ -578,8 +578,8 @@ func TestShreddedVariantWriteScalarTypes(t *testing.T) { } // TestShreddedVariantWriteMixedTypeField writes an object field that is int64 in most -// rows and string in a few. Inference shreds it as int64 (the majority); the minority -// string rows must round-trip through the unshredded residual, not be dropped. +// rows and string in a few. The mixed field is not uniform, so it is not shredded at +// all; every row must still round-trip through the unshredded value. func TestShreddedVariantWriteMixedTypeField(t *testing.T) { mem := memory.DefaultAllocator iceSchema := iceberg.NewSchema(0, @@ -633,7 +633,8 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) { files = append(files, df) } require.Len(t, files, 1) - assert.True(t, payloadHasTypedValue(t, files[0].FilePath()), "f shreds as int64 (majority)") + assert.False(t, payloadHasTypedValue(t, files[0].FilePath()), + "mixed-type field is not uniform, so payload must not shred") p := strings.TrimPrefix(files[0].FilePath(), "file://") f, err := os.Open(p) @@ -656,10 +657,10 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) { assert.JSONEqf(t, want, string(got), "row %d (mixed-type minority must round-trip)", i) } - // f shreds as int64 but the string rows land in the residual value column, so its - // typed stats do not cover all values: the bound must be dropped (Java value() rule). + // The mixed-type field is not shredded, so there is no typed_value to derive stats + // from: the field must not emit a variant bound. _, hasLower := files[0].LowerBoundValues()[2] - assert.False(t, hasLower, "partial-shred field must not emit a variant bound") + assert.False(t, hasLower, "unshredded field must not emit a variant bound") } // TestShreddedVariantWriteRowConservation verifies every row survives bootstrap -> From e66c3291e6e4f2fc0db84ff848f3f949ad5bf9bf Mon Sep 17 00:00:00 2001 From: Neelesh Salian Date: Wed, 19 Aug 2026 13:05:07 -0700 Subject: [PATCH 2/2] fold variant admission into incremental state, simplify tests --- table/internal/variant_shredding.go | 95 +++++++++++++----------- table/internal/variant_shredding_test.go | 21 ++---- 2 files changed, 59 insertions(+), 57 deletions(-) diff --git a/table/internal/variant_shredding.go b/table/internal/variant_shredding.go index 847014f2f4..c7c8670fbe 100644 --- a/table/internal/variant_shredding.go +++ b/table/internal/variant_shredding.go @@ -48,23 +48,25 @@ var decimalPriority = map[variant.Type]int{ } type fieldInfo struct { - typeCounts map[variant.Type]int + admitted variant.Type + hasAdmitted bool + mixed bool observationCount int maxDecimalScale int maxDecimalIntDigits int } func newFieldInfo() *fieldInfo { - return &fieldInfo{typeCounts: make(map[variant.Type]int)} + return &fieldInfo{} } -// observe records one value's type at this node (per-value counting). +// observe records one value at this node, folding its type into the admitted type. func (f *fieldInfo) observe(v variant.Value) { f.observationCount++ t := v.Type() // arrow-go has a single Bool type, so no false-folds-to-true step. - f.typeCounts[t]++ + f.admit(t) if isDecimalType(t) { intDigits, scale := decimalDigits(v.Value()) @@ -77,52 +79,59 @@ func (f *fieldInfo) observe(v variant.Value) { } } -// admittedType returns the single type this node shreds to. The integer and -// decimal families each collapse to their widest observed member; if more than one -// family remains the field is mixed-type and not shreddable (ok=false), mirroring -// Java VariantShreddingAnalyzer's admittedType. -func (f *fieldInfo) admittedType() (variant.Type, bool) { - var widestInt, widestDec, other variant.Type - haveInt, haveDec := false, false - otherFamilies := 0 - - for t := range f.typeCounts { - switch { - case isIntegerType(t): - if !haveInt || integerPriority[t] > integerPriority[widestInt] { - widestInt = t - } - haveInt = true - case isDecimalType(t): - if !haveDec || decimalPriority[t] > decimalPriority[widestDec] { - widestDec = t - } - haveDec = true - default: - other = t - otherFamilies++ - } +// admit widens the admitted type with t, or marks the node mixed when t belongs to +// a different family. +func (f *fieldInfo) admit(t variant.Type) { + if f.mixed { + return } + if !f.hasAdmitted { + f.admitted, f.hasAdmitted = t, true - families := otherFamilies - if haveInt { - families++ + return } - if haveDec { - families++ + merged, ok := mergeFamily(f.admitted, t) + if !ok { + f.mixed = true + + return } - if families != 1 { - return 0, false + f.admitted = merged +} + +// mergeFamily widens current with candidate within the integer or decimal family, +// returning ok=false when they belong to different families (mirroring Java's +// VariantShreddingAnalyzer.mergeFamily). +func mergeFamily(current, candidate variant.Type) (variant.Type, bool) { + if current == candidate { + return current, true } + if isIntegerType(current) && isIntegerType(candidate) { + if integerPriority[candidate] > integerPriority[current] { + return candidate, true + } - switch { - case haveInt: - return widestInt, true - case haveDec: - return widestDec, true - default: - return other, true + return current, true + } + if isDecimalType(current) && isDecimalType(candidate) { + if decimalPriority[candidate] > decimalPriority[current] { + return candidate, true + } + + return current, true + } + + return 0, false +} + +// admittedType returns the single type this node shreds to, or ok=false when the +// node saw no values or mixed more than one type family. +func (f *fieldInfo) admittedType() (variant.Type, bool) { + if !f.hasAdmitted || f.mixed { + return 0, false } + + return f.admitted, true } func isIntegerType(t variant.Type) bool { diff --git a/table/internal/variant_shredding_test.go b/table/internal/variant_shredding_test.go index a0300131c1..93f1cdeff7 100644 --- a/table/internal/variant_shredding_test.go +++ b/table/internal/variant_shredding_test.go @@ -171,22 +171,15 @@ func TestAnalyzeCrossFamilyNotShredded(t *testing.T) { // TestAnalyzeMixedSiblingKeepsUniform: a mixed-type field is dropped from the // shredded type while its uniform sibling still shreds. func TestAnalyzeMixedSiblingKeepsUniform(t *testing.T) { - var sample []variant.Value - for range 8 { - sample = append(sample, mkVar(t, map[string]any{"good": bigI64, "bad": bigI64})) - } - for range 2 { - sample = append(sample, mkVar(t, map[string]any{"good": bigI64, "bad": "s"})) - } + sample := mkVars(t, + map[string]any{"good": bigI64, "bad": bigI64}, + map[string]any{"good": bigI64, "bad": "s"}, + ) dt, ok := AnalyzeVariantShredding(sample) require.True(t, ok) - st := dt.(*arrow.StructType) - g, hasGood := st.FieldsByName("good") - require.True(t, hasGood, "uniform sibling must shred") - assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, g[0].Type), "good shreds as Int64, got %s", g[0].Type) - _, hasBad := st.FieldsByName("bad") - assert.False(t, hasBad, "mixed sibling must be dropped from the shredded type") + want := arrow.StructOf(arrow.Field{Name: "good", Type: arrow.PrimitiveTypes.Int64, Nullable: true}) + assert.Truef(t, arrow.TypeEqual(want, dt), "want %s got %s", want, dt) } // TestAnalyzeEmptyContainerNotShredded: an object with no fields and an array with no @@ -361,7 +354,7 @@ func TestInferredTypedScalarsShred(t *testing.T) { st := extensions.NewShreddedVariantType(inner) bldr := extensions.NewVariantBuilder(mem, st) defer bldr.Release() - bldr.Append(mkBuilt(t, c.build)) + bldr.Append(v) arr := bldr.NewArray().(*extensions.VariantArray) defer arr.Release()