diff --git a/table/internal/variant_shredding.go b/table/internal/variant_shredding.go index dd0c90d5f0..c7c8670fbe 100644 --- a/table/internal/variant_shredding.go +++ b/table/internal/variant_shredding.go @@ -18,7 +18,6 @@ package internal import ( - "slices" "sort" "github.com/apache/arrow-go/v18/arrow" @@ -27,10 +26,11 @@ import ( "github.com/apache/arrow-go/v18/parquet/variant" ) -// Variant shredding inference: most-common-type selection with explicit -// tie-break, integer/decimal widening, and frequency/depth/field-count bounds. -// Field caps are per-object-node (matching Java); no global budget - total is -// bounded by the sample, maxShreddingDepth, and the per-node caps. +// Variant shredding inference: a field shreds only when all its non-null values +// share one type family (the integer and decimal families widen to their widest +// observed member); a field mixing more than one family is not shreddable and +// stays in the residual value. Frequency/depth/field-count bounds and per-object- +// node caps match Java; no global budget. const ( minFieldFrequency = 0.10 maxShreddedFields = 300 @@ -47,35 +47,26 @@ var decimalPriority = map[variant.Type]int{ variant.Decimal4: 0, variant.Decimal8: 1, variant.Decimal16: 2, } -// tieBreakPriority breaks count ties in most-common-type selection; higher wins. -// Types absent here (Null, Array, Object) resolve to -1. -var tieBreakPriority = map[variant.Type]int{ - variant.Bool: 0, variant.Int8: 1, variant.Int16: 2, variant.Int32: 3, - variant.Int64: 4, variant.Float: 5, variant.Double: 6, variant.Decimal4: 7, - variant.Decimal8: 8, variant.Decimal16: 9, variant.Date: 10, variant.Time: 11, - variant.TimestampMicros: 12, variant.TimestampMicrosNTZ: 13, variant.Binary: 14, - variant.String: 15, variant.TimestampNanos: 16, variant.TimestampNanosNTZ: 17, - variant.UUID: 18, -} - type fieldInfo struct { - typeCounts map[variant.Type]int + admitted variant.Type + hasAdmitted bool + mixed bool observationCount int maxDecimalScale int maxDecimalIntDigits int } func newFieldInfo() *fieldInfo { - return &fieldInfo{typeCounts: make(map[variant.Type]int)} + return &fieldInfo{} } -// observe records one value's type at this node (per-value counting). +// observe records one value at this node, folding its type into the admitted type. func (f *fieldInfo) observe(v variant.Value) { f.observationCount++ t := v.Type() // arrow-go has a single Bool type, so no false-folds-to-true step. - f.typeCounts[t]++ + f.admit(t) if isDecimalType(t) { intDigits, scale := decimalDigits(v.Value()) @@ -88,67 +79,59 @@ func (f *fieldInfo) observe(v variant.Value) { } } -// mostCommonType collapses int/decimal families to the widest, then picks max by count. -func (f *fieldInfo) mostCommonType() (variant.Type, bool) { - combined := make(map[variant.Type]int) - - intTotal, decTotal := 0, 0 - var widestInt, widestDec variant.Type - haveInt, haveDec := false, false - - for t, c := range f.typeCounts { - switch { - case isIntegerType(t): - intTotal += c - if !haveInt || integerPriority[t] > integerPriority[widestInt] { - widestInt, haveInt = t, true - } - case isDecimalType(t): - decTotal += c - if !haveDec || decimalPriority[t] > decimalPriority[widestDec] { - widestDec, haveDec = t, true - } - default: - combined[t] = c - } +// admit widens the admitted type with t, or marks the node mixed when t belongs to +// a different family. +func (f *fieldInfo) admit(t variant.Type) { + if f.mixed { + return } - if haveInt { - combined[widestInt] = intTotal + if !f.hasAdmitted { + f.admitted, f.hasAdmitted = t, true + + return } - if haveDec { - combined[widestDec] = decTotal + merged, ok := mergeFamily(f.admitted, t) + if !ok { + f.mixed = true + + return } + f.admitted = merged +} - if len(combined) == 0 { - return 0, false +// mergeFamily widens current with candidate within the integer or decimal family, +// returning ok=false when they belong to different families (mirroring Java's +// VariantShreddingAnalyzer.mergeFamily). +func mergeFamily(current, candidate variant.Type) (variant.Type, bool) { + if current == candidate { + return current, true } + if isIntegerType(current) && isIntegerType(candidate) { + if integerPriority[candidate] > integerPriority[current] { + return candidate, true + } - // Max by count, then tieBreakPriority, then type value (stable, sorted keys). - var best variant.Type - bestCount, bestPrio := -1, -2 - first := true - keys := make([]variant.Type, 0, len(combined)) - for t := range combined { - keys = append(keys, t) + return current, true } - slices.Sort(keys) - for _, t := range keys { - c := combined[t] - p := tiePriority(t) - if first || c > bestCount || (c == bestCount && p > bestPrio) { - best, bestCount, bestPrio, first = t, c, p, false + if isDecimalType(current) && isDecimalType(candidate) { + if decimalPriority[candidate] > decimalPriority[current] { + return candidate, true } + + return current, true } - return best, true + return 0, false } -func tiePriority(t variant.Type) int { - if p, ok := tieBreakPriority[t]; ok { - return p +// admittedType returns the single type this node shreds to, or ok=false when the +// node saw no values or mixed more than one type family. +func (f *fieldInfo) admittedType() (variant.Type, bool) { + if !f.hasAdmitted || f.mixed { + return 0, false } - return -1 + return f.admitted, true } func isIntegerType(t variant.Type) bool { @@ -213,7 +196,7 @@ func AnalyzeVariantShredding(sample []variant.Value) (arrow.DataType, bool) { traverseVariant(root, v, 0) } - rootType, ok := root.info.mostCommonType() + rootType, ok := root.info.admittedType() if !ok { return nil, false } @@ -324,7 +307,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType { fields := make([]arrow.Field, 0, len(names)) for _, name := range names { child := node.objectChildren[name] - ct, ok := child.info.mostCommonType() + ct, ok := child.info.admittedType() if !ok { continue } @@ -343,7 +326,7 @@ func buildInnerType(node *pathNode, t variant.Type) arrow.DataType { if node.arrayElement == nil { return nil } - et, ok := node.arrayElement.info.mostCommonType() + et, ok := node.arrayElement.info.admittedType() if !ok { return nil } diff --git a/table/internal/variant_shredding_test.go b/table/internal/variant_shredding_test.go index d1754b88cc..93f1cdeff7 100644 --- a/table/internal/variant_shredding_test.go +++ b/table/internal/variant_shredding_test.go @@ -53,6 +53,18 @@ func mkVars(t *testing.T, raws ...any) []variant.Value { return out } +// mkBuilt builds a variant.Value via a builder callback, for leaf types (decimal, +// timestamp) that the reflection-based mkVar cannot express directly. +func mkBuilt(t *testing.T, build func(*variant.Builder) error) variant.Value { + t.Helper() + var b variant.Builder + require.NoError(t, build(&b)) + v, err := b.Build() + require.NoError(t, err) + + return v +} + // bigI64 is large enough to force Int64 encoding (> 2^32). const bigI64 = int64(5_000_000_000) @@ -115,8 +127,10 @@ func TestAnalyzeIntegerWidening(t *testing.T) { "widening must pick Int64, got %s", f[0].Type) } -func TestAnalyzeMixedTypeMajority(t *testing.T) { - // "v" is an int in 7 rows, a string in 3 -> majority int wins. +// TestAnalyzeMixedTypeNotShredded: a field mixing type families (int in 7 rows, +// string in 3) is not uniform and must not shred - it is this object's only +// field, so nothing shreds at all. +func TestAnalyzeMixedTypeNotShredded(t *testing.T) { var sample []variant.Value for range 7 { sample = append(sample, mkVar(t, map[string]any{"v": bigI64})) @@ -124,13 +138,78 @@ func TestAnalyzeMixedTypeMajority(t *testing.T) { for range 3 { sample = append(sample, mkVar(t, map[string]any{"v": "s"})) } + _, ok := AnalyzeVariantShredding(sample) + assert.False(t, ok, "mixed int+string field must not shred") +} + +// TestAnalyzeCrossFamilyNotShredded: distinct scalar families that do not widen +// into each other are not uniform and must not shred. +func TestAnalyzeCrossFamilyNotShredded(t *testing.T) { + microsTS := func(b *variant.Builder) error { + return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), true, true) + } + nanosTS := func(b *variant.Builder) error { + return b.AppendTimestamp(arrow.Timestamp(1_700_000_000_000_000), false, true) + } + dec := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(12345)) } + + for _, c := range []struct { + name string + sample []variant.Value + }{ + {"float+double", mkVars(t, float32(1.5), float64(2.5))}, + {"timestamp micros+nanos", []variant.Value{mkBuilt(t, microsTS), mkBuilt(t, nanosTS)}}, + {"int+decimal", []variant.Value{mkVar(t, bigI64), mkBuilt(t, dec)}}, + } { + t.Run(c.name, func(t *testing.T) { + _, ok := AnalyzeVariantShredding(c.sample) + assert.Falsef(t, ok, "%s must not shred", c.name) + }) + } +} + +// TestAnalyzeMixedSiblingKeepsUniform: a mixed-type field is dropped from the +// shredded type while its uniform sibling still shreds. +func TestAnalyzeMixedSiblingKeepsUniform(t *testing.T) { + sample := mkVars(t, + map[string]any{"good": bigI64, "bad": bigI64}, + map[string]any{"good": bigI64, "bad": "s"}, + ) dt, ok := AnalyzeVariantShredding(sample) require.True(t, ok) - st := dt.(*arrow.StructType) - f, _ := st.FieldsByName("v") - require.Len(t, f, 1) - assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, f[0].Type), - "majority int must win, got %s", f[0].Type) + + want := arrow.StructOf(arrow.Field{Name: "good", Type: arrow.PrimitiveTypes.Int64, Nullable: true}) + assert.Truef(t, arrow.TypeEqual(want, dt), "want %s got %s", want, dt) +} + +// TestAnalyzeEmptyContainerNotShredded: an object with no fields and an array with no +// elements have nothing to shred, so inference must decline. +func TestAnalyzeEmptyContainerNotShredded(t *testing.T) { + _, ok := AnalyzeVariantShredding(mkVars(t, map[string]any{}, map[string]any{})) + assert.False(t, ok, "empty object must not shred") + + _, ok = AnalyzeVariantShredding(mkVars(t, []any{}, []any{})) + assert.False(t, ok, "empty array must not shred") +} + +// TestAnalyzeDecimalWidening: the decimal family widens to a single leaf and still +// shreds; only cross-family mixing blocks shredding. +func TestAnalyzeDecimalWidening(t *testing.T) { + dec4 := func(b *variant.Builder) error { return b.AppendDecimal4(2, decimal.Decimal32(12345)) } + dec8 := func(b *variant.Builder) error { return b.AppendDecimal8(2, decimal.Decimal64(1234567890)) } + dt, ok := AnalyzeVariantShredding([]variant.Value{mkBuilt(t, dec4), mkBuilt(t, dec8)}) + require.True(t, ok, "decimal family must still shred after widening") + _, isDec := dt.(*arrow.Decimal128Type) + assert.Truef(t, isDec, "decimal widening must produce a decimal leaf, got %s", dt) +} + +// TestAnalyzeNullsPlusOneType: variant-null values are not observed, so a field +// null in some rows and one type in the rest still shreds as that type. +func TestAnalyzeNullsPlusOneType(t *testing.T) { + dt, ok := AnalyzeVariantShredding(mkVars(t, nil, bigI64, nil, bigI64)) + require.True(t, ok) + assert.Truef(t, arrow.TypeEqual(arrow.PrimitiveTypes.Int64, dt), + "nulls + int must shred as Int64, got %s", dt) } func TestAnalyzeScalarRoot(t *testing.T) { @@ -268,22 +347,14 @@ func TestInferredTypedScalarsShred(t *testing.T) { } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - mk := func() variant.Value { - var b variant.Builder - require.NoError(t, c.build(&b)) - v, err := b.Build() - require.NoError(t, err) - - return v - } - v := mk() + v := mkBuilt(t, c.build) inner, ok := AnalyzeVariantShredding([]variant.Value{v}) require.Truef(t, ok, "%s should infer a shredded type", c.name) st := extensions.NewShreddedVariantType(inner) bldr := extensions.NewVariantBuilder(mem, st) defer bldr.Release() - bldr.Append(mk()) + bldr.Append(v) arr := bldr.NewArray().(*extensions.VariantArray) defer arr.Release() diff --git a/table/variant_shredded_write_test.go b/table/variant_shredded_write_test.go index 9e4b8ec92e..7c8f639e36 100644 --- a/table/variant_shredded_write_test.go +++ b/table/variant_shredded_write_test.go @@ -578,8 +578,8 @@ func TestShreddedVariantWriteScalarTypes(t *testing.T) { } // TestShreddedVariantWriteMixedTypeField writes an object field that is int64 in most -// rows and string in a few. Inference shreds it as int64 (the majority); the minority -// string rows must round-trip through the unshredded residual, not be dropped. +// rows and string in a few. The mixed field is not uniform, so it is not shredded at +// all; every row must still round-trip through the unshredded value. func TestShreddedVariantWriteMixedTypeField(t *testing.T) { mem := memory.DefaultAllocator iceSchema := iceberg.NewSchema(0, @@ -633,7 +633,8 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) { files = append(files, df) } require.Len(t, files, 1) - assert.True(t, payloadHasTypedValue(t, files[0].FilePath()), "f shreds as int64 (majority)") + assert.False(t, payloadHasTypedValue(t, files[0].FilePath()), + "mixed-type field is not uniform, so payload must not shred") p := strings.TrimPrefix(files[0].FilePath(), "file://") f, err := os.Open(p) @@ -656,10 +657,10 @@ func TestShreddedVariantWriteMixedTypeField(t *testing.T) { assert.JSONEqf(t, want, string(got), "row %d (mixed-type minority must round-trip)", i) } - // f shreds as int64 but the string rows land in the residual value column, so its - // typed stats do not cover all values: the bound must be dropped (Java value() rule). + // The mixed-type field is not shredded, so there is no typed_value to derive stats + // from: the field must not emit a variant bound. _, hasLower := files[0].LowerBoundValues()[2] - assert.False(t, hasLower, "partial-shred field must not emit a variant bound") + assert.False(t, hasLower, "unshredded field must not emit a variant bound") } // TestShreddedVariantWriteRowConservation verifies every row survives bootstrap ->