diff --git a/go.mod b/go.mod index 60723a7a5e..3169b1a214 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 github.com/RoaringBitmap/roaring/v2 v2.26.0 github.com/alexflint/go-arg v1.6.1 - github.com/apache/arrow-go/v18 v18.7.0 + github.com/apache/arrow-go/v18 v18.8.0 github.com/aws/aws-sdk-go-v2 v1.45.1 github.com/aws/aws-sdk-go-v2/config v1.33.1 github.com/aws/aws-sdk-go-v2/credentials v1.20.1 @@ -95,7 +95,7 @@ require ( github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect github.com/alexflint/go-scalar v1.2.0 // indirect - github.com/andybalholm/brotli v1.2.2 // indirect + github.com/andybalholm/brotli v1.2.3 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect @@ -184,7 +184,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mattn/go-shellwords v1.0.13 // indirect github.com/mattn/go-sqlite3 v1.14.34 // indirect @@ -276,9 +276,9 @@ require ( google.golang.org/protobuf v1.36.12 // indirect gopkg.in/ini.v1 v1.67.3 // indirect gotest.tools/v3 v3.5.2 // indirect - modernc.org/libc v1.73.4 // indirect + modernc.org/libc v1.74.4 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.53.0 // indirect + modernc.org/sqlite v1.57.0 // indirect tags.cncf.io/container-device-interface v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 7285623b08..578d420b22 100644 --- a/go.sum +++ b/go.sum @@ -95,12 +95,12 @@ github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+W github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o= github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30= github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= -github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= -github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ= +github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= -github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY= -github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A= +github.com/apache/arrow-go/v18 v18.8.0 h1:BLOzbPv7bxMPgXPacAg6HQjnxupYsZzC4tf+FkqPU/M= +github.com/apache/arrow-go/v18 v18.8.0/go.mod h1:uJCFfCwq0KsxCmsCfQg4ft+LsW+iHYzAXiSDh5ug/8U= github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= @@ -363,8 +363,8 @@ github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQE github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= -github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -440,8 +440,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= @@ -766,7 +766,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -824,20 +823,20 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c= -modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= -modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws= -modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc= -modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA= -modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -846,8 +845,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M= -modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/table/arrow_utils.go b/table/arrow_utils.go index ef247f0c8b..a6e1c1e29e 100644 --- a/table/arrow_utils.go +++ b/table/arrow_utils.go @@ -436,7 +436,7 @@ func (c convertToIceberg) Primitive(dt arrow.DataType) (result iceberg.NestedFie switch dt.ExtensionName() { case "arrow.uuid": result.Type = iceberg.PrimitiveTypes.UUID - case "parquet.variant": + case extensions.VariantExtensionName, extensions.LegacyVariantExtensionName: result.Type = iceberg.VariantType{} case "geoarrow.wkb": wkb, ok := dt.(*geoarrow.WKBType) diff --git a/table/arrow_utils_test.go b/table/arrow_utils_test.go index 71566ed3d5..580847211e 100644 --- a/table/arrow_utils_test.go +++ b/table/arrow_utils_test.go @@ -389,7 +389,7 @@ func TestVariantArrowConversion(t *testing.T) { dataField := arrowSc.Field(1) ext, ok := dataField.Type.(arrow.ExtensionType) require.True(t, ok, "expected extension type, got %T", dataField.Type) - assert.Equal(t, "parquet.variant", ext.ExtensionName()) + assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName()) ice, err := table.ArrowSchemaToIceberg(arrowSc, false, nil) require.NoError(t, err) @@ -427,7 +427,7 @@ func TestVariantArrowConversion(t *testing.T) { for _, idx := range []int{1, 2} { ext, ok := arrowSc.Field(idx).Type.(arrow.ExtensionType) require.True(t, ok, "field %d should be extension type", idx) - assert.Equal(t, "parquet.variant", ext.ExtensionName()) + assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName()) } ice, err := table.ArrowSchemaToIceberg(arrowSc, false, nil) @@ -1199,28 +1199,28 @@ func ArrowRecordWithAllTimestampPrec(mem memory.Allocator) arrow.RecordBatch { batch, _, err := array.RecordFromJSON(mem, ArrowSchemaWithAllTimestampPrec, strings.NewReader(`[ { - "timestamp_s": "2023-01-01T19:25:00-05:00", + "timestamp_s": "2023-01-01T19:25:00", "timestamptz_s": "2023-01-01T19:25:00Z", - "timestamp_ms": "2023-01-01T19:25:00.123-05:00", + "timestamp_ms": "2023-01-01T19:25:00.123", "timestamptz_ms": "2023-01-01T19:25:00.123Z", - "timestamp_us": "2023-01-01T19:25:00.123456-05:00", + "timestamp_us": "2023-01-01T19:25:00.123456", "timestamptz_us": "2023-01-01T19:25:00.123456Z", - "timestamp_ns": "2024-07-11T03:30:00.123456789-05:00", + "timestamp_ns": "2024-07-11T03:30:00.123456789", "timestamptz_ns": "2023-01-01T19:25:00.123456789Z", "timestamptz_us_etc_utc": "2023-01-01T19:25:00.123456Z", "timestamptz_ns_z": "2024-07-11T03:30:00.123456789Z", "timestamptz_s_0000": "2023-01-01T19:25:00Z" }, {}, { - "timestamp_s": "2023-03-01T19:25:00-05:00", + "timestamp_s": "2023-03-01T19:25:00", "timestamptz_s": "2023-03-01T19:25:00Z", - "timestamp_ms": "2023-03-01T19:25:00.123-05:00", + "timestamp_ms": "2023-03-01T19:25:00.123", "timestamptz_ms": "2023-03-01T19:25:00.123Z", - "timestamp_us": "2023-03-01T19:25:00.123456-05:00", + "timestamp_us": "2023-03-01T19:25:00.123456", "timestamptz_us": "2023-03-01T19:25:00.123456Z", - "timestamp_ns": "2024-07-11T03:30:00.9876543210-05:00", - "timestamptz_ns": "2023-03-01T19:25:00.9876543210Z", + "timestamp_ns": "2024-07-11T03:30:00.987654321", + "timestamptz_ns": "2023-03-01T19:25:00.987654321Z", "timestamptz_us_etc_utc": "2023-03-01T19:25:00.123456Z", - "timestamptz_ns_z": "2024-07-11T03:30:00.9876543210Z", + "timestamptz_ns_z": "2024-07-11T03:30:00.987654321Z", "timestamptz_s_0000": "2023-03-01T19:25:00Z" } ]`)) diff --git a/table/equality_delete_reader_test.go b/table/equality_delete_reader_test.go index b022b95cca..412dde78da 100644 --- a/table/equality_delete_reader_test.go +++ b/table/equality_delete_reader_test.go @@ -811,11 +811,11 @@ func TestEqualityDeleteNullableFastPathKeys(t *testing.T) { keyType: iceberg.PrimitiveTypes.Timestamp, dataJSON: `[ {"row_id": 1, "key": null}, - {"row_id": 2, "key": "1970-01-01T00:00:00.000000Z"}, - {"row_id": 3, "key": "2024-01-02T03:04:05.000000Z"} + {"row_id": 2, "key": "1970-01-01T00:00:00.000000"}, + {"row_id": 3, "key": "2024-01-02T03:04:05.000000"} ]`, nullDeleteJSON: `[{"key": null}]`, - zeroDeleteJSON: `[{"key": "1970-01-01T00:00:00.000000Z"}]`, + zeroDeleteJSON: `[{"key": "1970-01-01T00:00:00.000000"}]`, }, } @@ -880,7 +880,7 @@ func TestEqualityDeleteNullableCompositeFastPathKey(t *testing.T) { name: "partially null key", dataJSON: `[ {"row_id": 1, "id": 7, "deleted_at": null}, - {"row_id": 2, "id": 7, "deleted_at": "1970-01-01T00:00:00.000000Z"}, + {"row_id": 2, "id": 7, "deleted_at": "1970-01-01T00:00:00.000000"}, {"row_id": 3, "id": 8, "deleted_at": null}, {"row_id": 4, "id": null, "deleted_at": null} ]`, @@ -891,7 +891,7 @@ func TestEqualityDeleteNullableCompositeFastPathKey(t *testing.T) { name: "all-null key", dataJSON: `[ {"row_id": 1, "id": null, "deleted_at": null}, - {"row_id": 2, "id": null, "deleted_at": "1970-01-01T00:00:00.000000Z"}, + {"row_id": 2, "id": null, "deleted_at": "1970-01-01T00:00:00.000000"}, {"row_id": 3, "id": 7, "deleted_at": null} ]`, deleteJSON: `[{"id": null, "deleted_at": null}]`, diff --git a/table/internal/parquet_files.go b/table/internal/parquet_files.go index d8071c921f..94f46e968b 100644 --- a/table/internal/parquet_files.go +++ b/table/internal/parquet_files.go @@ -613,13 +613,13 @@ func (p parquetFormat) NewFileWriter(ctx context.Context, fs iceio.WriteFileIO, counter := &internal.CountingWriter{W: fw} mem := compute.GetAllocator(ctx) - writerProps, err := getWriteProperties(info.WriteProps, arrowSchema) + arrProps := writeArrowProps(mem) + writerProps, err := getWriteProperties(info.WriteProps, arrowSchema, arrProps) if err != nil { fw.Close() return nil, err } - arrProps := pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithStoreSchema()) writer, err := pqarrow.NewFileWriter(arrowSchema, counter, writerProps, arrProps) if err != nil { @@ -691,9 +691,13 @@ func typeHasDecimal128(dt arrow.DataType) bool { return false } +func writeArrowProps(mem memory.Allocator) pqarrow.ArrowWriterProperties { + return pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithStoreSchema()) +} + // getWriteProperties requires explicit write properties so misconfigured writer // plumbing fails fast instead of silently defaulting Parquet settings. -func getWriteProperties(writeProps any, arrowSchema *arrow.Schema) (*parquet.WriterProperties, error) { +func getWriteProperties(writeProps any, arrowSchema *arrow.Schema, arrProps pqarrow.ArrowWriterProperties) (*parquet.WriterProperties, error) { if writeProps == nil { return nil, fmt.Errorf("%w: write properties are required", iceberg.ErrInvalidArgument) } @@ -711,9 +715,103 @@ func getWriteProperties(writeProps any, arrowSchema *arrow.Schema) (*parquet.Wri wp = append(wp, parquet.WithStoreDecimalAsInteger(true)) } + // Match Iceberg Java: apply parquet-mr's cost-based dictionary fallback to every leaf + // column so high-cardinality columns fall back to PLAIN rather than keeping a dictionary. + // arrow-go otherwise enables it only for uncompressed columns, so zstd (our default) would + // retain dictionaries on all-distinct columns and roughly double their size. + costFallback, err := dictCostFallbackProps(arrowSchema, wp, arrProps) + if err != nil { + return nil, err + } + wp = append(wp, costFallback...) + return parquet.NewWriterProperties(wp...), nil } +// dictCostFallbackProps returns a WithDictionaryCostFallbackFor(true) property per leaf, walking the arrow schema directly (extensions unwrapped) and falling back to pqarrow.ToParquet for list/map schemas. +func dictCostFallbackProps(arrowSchema *arrow.Schema, base []parquet.WriterProperty, arrProps pqarrow.ArrowWriterProperties) ([]parquet.WriterProperty, error) { + if schemaHasListOrMap(arrowSchema) { + return dictCostFallbackViaParquet(arrowSchema, base, arrProps) + } + + var props []parquet.WriterProperty + var walk func(prefix string, dt arrow.DataType) + walk = func(prefix string, dt arrow.DataType) { + if ext, ok := dt.(arrow.ExtensionType); ok { + dt = ext.StorageType() + } + if d, ok := dt.(*arrow.DictionaryType); ok { + dt = d.ValueType + } + if r, ok := dt.(*arrow.RunEndEncodedType); ok { + dt = r.Encoded() + } + if st, ok := dt.(*arrow.StructType); ok { + for _, f := range st.Fields() { + walk(prefix+"."+f.Name, f.Type) + } + + return + } + props = append(props, parquet.WithDictionaryCostFallbackFor(prefix, true)) + } + for _, f := range arrowSchema.Fields() { + walk(f.Name, f.Type) + } + + return props, nil +} + +// dictCostFallbackViaParquet is the authoritative path for list/map schemas, whose parquet leaf naming the direct walk does not reproduce. +func dictCostFallbackViaParquet(arrowSchema *arrow.Schema, base []parquet.WriterProperty, arrProps pqarrow.ArrowWriterProperties) ([]parquet.WriterProperty, error) { + parquetSchema, err := pqarrow.ToParquet(arrowSchema, parquet.NewWriterProperties(base...), arrProps) + if err != nil { + return nil, err + } + + props := make([]parquet.WriterProperty, 0, parquetSchema.NumColumns()) + for i := range parquetSchema.NumColumns() { + props = append(props, parquet.WithDictionaryCostFallbackFor(parquetSchema.Column(i).Path(), true)) + } + + return props, nil +} + +func schemaHasListOrMap(sc *arrow.Schema) bool { + var has func(dt arrow.DataType) bool + has = func(dt arrow.DataType) bool { + if ext, ok := dt.(arrow.ExtensionType); ok { + dt = ext.StorageType() + } + if d, ok := dt.(*arrow.DictionaryType); ok { + dt = d.ValueType + } + if r, ok := dt.(*arrow.RunEndEncodedType); ok { + dt = r.Encoded() + } + switch t := dt.(type) { + case *arrow.ListType, *arrow.LargeListType, *arrow.FixedSizeListType, + *arrow.ListViewType, *arrow.LargeListViewType, *arrow.MapType: + return true + case *arrow.StructType: + for _, f := range t.Fields() { + if has(f.Type) { + return true + } + } + } + + return false + } + for _, f := range sc.Fields() { + if has(f.Type) { + return true + } + } + + return false +} + // Write appends a record batch to the Parquet file. func (w *ParquetFileWriter) Write(batch arrow.RecordBatch) error { writeBatch, err := w.normalizeGeoBatch(batch) @@ -2371,7 +2469,7 @@ func (p *pruneParquetSchema) Field(field pqarrow.SchemaField, result arrow.Field // Variant is an extension type wrapping a struct (metadata + value). // Select the entire field including all children. - if ext, ok := field.Field.Type.(arrow.ExtensionType); ok && ext.ExtensionName() == "parquet.variant" { + if ext, ok := field.Field.Type.(arrow.ExtensionType); ok && extensions.IsVariantExtensionName(ext.ExtensionName()) { return p.projectVariant(field) } diff --git a/table/internal/parquet_files_internal_test.go b/table/internal/parquet_files_internal_test.go index 2d06ab58e1..a665f8c50a 100644 --- a/table/internal/parquet_files_internal_test.go +++ b/table/internal/parquet_files_internal_test.go @@ -23,6 +23,10 @@ import ( "testing" "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/extensions" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet" + "github.com/apache/arrow-go/v18/parquet/pqarrow" "github.com/apache/iceberg-go" iceio "github.com/apache/iceberg-go/io" "github.com/geoarrow/geoarrow-go" @@ -144,3 +148,103 @@ func TestNewFileWriterUsesProvidedSchemaMetadata(t *testing.T) { assert.Equal(t, variantFieldIDs, parquetWriter.variantFieldIDs) assert.Equal(t, 99, parquetWriter.colMapping["provided"]) } + +// getWriteProperties must enable the cost-based dictionary fallback for every leaf column +// (parquet-mr parity); arrow-go leaves it off for compressed columns, and Iceberg writes zstd. +func TestGetWritePropertiesEnablesDictCostFallback(t *testing.T) { + arrowSchema := arrow.NewSchema([]arrow.Field{ + {Name: "s", Type: arrow.BinaryTypes.String, Nullable: true}, + {Name: "n", Type: arrow.PrimitiveTypes.Int64, Nullable: true}, + }, nil) + + format := parquetFormat{} + wp, err := getWriteProperties(format.GetWriteProperties(iceberg.Properties{}), arrowSchema, writeArrowProps(memory.DefaultAllocator)) + require.NoError(t, err) + + assert.True(t, wp.DictionaryCostFallbackEnabledFor("s")) + assert.True(t, wp.DictionaryCostFallbackEnabledFor("n")) +} + +// TestDictCostFallbackWalkMatchesToParquet locks the arrow-leaf walk to pqarrow.ToParquet's leaf paths (flat, nested, variant, decimal). +func TestDictCostFallbackWalkMatchesToParquet(t *testing.T) { + variantType := extensions.NewShreddedVariantType(arrow.StructOf( + arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, + arrow.Field{Name: "b", Type: arrow.BinaryTypes.String}, + )) + schemas := map[string]*arrow.Schema{ + "flat": arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "s", Type: arrow.BinaryTypes.String}, + }, nil), + "nested": arrow.NewSchema([]arrow.Field{ + {Name: "w", Type: arrow.StructOf( + arrow.Field{Name: "x", Type: arrow.PrimitiveTypes.Int64}, + arrow.Field{Name: "y", Type: arrow.StructOf(arrow.Field{Name: "z", Type: arrow.BinaryTypes.String})}, + )}, + }, nil), + "variant": arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "payload", Type: variantType}, + }, nil), + "decimal": arrow.NewSchema([]arrow.Field{ + {Name: "d", Type: &arrow.Decimal128Type{Precision: 10, Scale: 2}}, + }, nil), + "dict": arrow.NewSchema([]arrow.Field{ + {Name: "c", Type: &arrow.DictionaryType{IndexType: arrow.PrimitiveTypes.Int32, ValueType: arrow.BinaryTypes.String}}, + }, nil), + "null": arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "z", Type: arrow.Null, Nullable: true}, + }, nil), + } + + for name, sc := range schemas { + t.Run(name, func(t *testing.T) { + require.False(t, schemaHasListOrMap(sc), "walk path should apply") + ps, err := pqarrow.ToParquet(sc, parquet.NewWriterProperties(), pqarrow.DefaultWriterProps()) + require.NoError(t, err) + + walk, err := dictCostFallbackProps(sc, nil, writeArrowProps(memory.DefaultAllocator)) + require.NoError(t, err) + wp := parquet.NewWriterProperties(walk...) + + require.Equal(t, ps.NumColumns(), len(walk), "leaf count differs from pqarrow") + for i := range ps.NumColumns() { + path := ps.Column(i).Path() + require.Truef(t, wp.DictionaryCostFallbackEnabledFor(path), "walk missing leaf path %q", path) + } + }) + } +} + +// TestDictCostFallbackListSchemaUsesToParquet exercises the list/map branch (dictCostFallbackViaParquet). +func TestDictCostFallbackListSchemaUsesToParquet(t *testing.T) { + sc := arrow.NewSchema([]arrow.Field{ + {Name: "id", Type: arrow.PrimitiveTypes.Int64}, + {Name: "tags", Type: arrow.ListOf(arrow.BinaryTypes.String)}, + }, nil) + require.True(t, schemaHasListOrMap(sc), "list schema must route to ToParquet") + + ps, err := pqarrow.ToParquet(sc, parquet.NewWriterProperties(), pqarrow.DefaultWriterProps()) + require.NoError(t, err) + props, err := dictCostFallbackProps(sc, nil, writeArrowProps(memory.DefaultAllocator)) + require.NoError(t, err) + wp := parquet.NewWriterProperties(props...) + + require.Equal(t, ps.NumColumns(), len(props)) + for i := range ps.NumColumns() { + require.Truef(t, wp.DictionaryCostFallbackEnabledFor(ps.Column(i).Path()), "missing leaf path %q", ps.Column(i).Path()) + } +} + +func TestDictCostFallbackWalkRunEndEncoded(t *testing.T) { + sc := arrow.NewSchema([]arrow.Field{ + {Name: "r", Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int32, arrow.BinaryTypes.String)}, + }, nil) + require.False(t, schemaHasListOrMap(sc), "rle-over-primitive must take the walk path") + + walk, err := dictCostFallbackProps(sc, nil, writeArrowProps(memory.DefaultAllocator)) + require.NoError(t, err) + require.Len(t, walk, 1, "rle unwraps to a single encoded leaf") + require.True(t, parquet.NewWriterProperties(walk...).DictionaryCostFallbackEnabledFor("r")) +} diff --git a/table/internal/parquet_files_test.go b/table/internal/parquet_files_test.go index 1c705c743a..7a30bc06ae 100644 --- a/table/internal/parquet_files_test.go +++ b/table/internal/parquet_files_test.go @@ -213,7 +213,7 @@ func constructTestTablePrimitiveTypes(t *testing.T) (*metadata.FileMetaData, tab "dates": "2022-01-02", "times": "17:30:34", "timestamps": "2022-01-02T17:30:34.399", - "timestamptzs": "2022-01-02T17:30:34.399", + "timestamptzs": "2022-01-02T17:30:34.399Z", "strings": "hello", "uuids": "`+uuid.NewMD5(uuid.NameSpaceDNS, []byte("foo")).String()+`", "binaries": "aGVsbG8=", @@ -230,7 +230,7 @@ func constructTestTablePrimitiveTypes(t *testing.T) (*metadata.FileMetaData, tab "dates": "2023-02-04", "times": "13:21:04", "timestamps": "2023-02-04T13:21:04.354", - "timestamptzs": "2023-02-04T13:21:04.354", + "timestamptzs": "2023-02-04T13:21:04.354Z", "strings": "world", "uuids": "`+uuid.NewMD5(uuid.NameSpaceDNS, []byte("bar")).String()+`", "binaries": "d29ybGQ=", @@ -1679,6 +1679,8 @@ func writeDictTestColumn(t *testing.T, tableProps iceberg.Properties, values []p format := internal.GetFileFormat(iceberg.ParquetFile) writeProps := format.GetWriteProperties(tableProps).([]parquet.WriterProperty) + // Mirror getWriteProperties: Iceberg enables the cost-based dictionary fallback per column. + writeProps = append(writeProps, parquet.WithDictionaryCostFallbackFor(dictTestColumn, true)) root, err := schema.NewGroupNode("schema", parquet.Repetitions.Required, schema.FieldList{ schema.NewByteArrayNode(dictTestColumn, parquet.Repetitions.Required, -1), @@ -2588,7 +2590,7 @@ func TestShreddedVariantReadRoundTrip(t *testing.T) { ext, ok := arrowSc.Field(0).Type.(arrow.ExtensionType) require.True(t, ok, "expected extension type, got %T", arrowSc.Field(0).Type) - assert.Equal(t, "parquet.variant", ext.ExtensionName()) + assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName()) // The shredded layout collapses back to a plain VariantType in Iceberg. iceSc, err := table.ArrowSchemaToIceberg(arrowSc, false, nil) diff --git a/table/table_test.go b/table/table_test.go index e573f87c5e..0d18778279 100644 --- a/table/table_test.go +++ b/table/table_test.go @@ -568,13 +568,13 @@ func (t *TableWritingTestSuite) TestAddFilesUnpartitioned() { Operation: table.OpAppend, Properties: iceberg.Properties{ "added-data-files": "5", - "added-files-size": "3070", + "added-files-size": "3080", "added-records": "5", "iceberg-version": "Apache Iceberg Go " + iceberg.Version(), "total-data-files": "5", "total-delete-files": "0", "total-equality-deletes": "0", - "total-files-size": "3070", + "total-files-size": "3080", "total-position-deletes": "0", "total-records": "5", }, @@ -790,14 +790,14 @@ func (t *TableWritingTestSuite) TestAddFilesPartitionedTable() { Operation: table.OpAppend, Properties: iceberg.Properties{ "added-data-files": "5", - "added-files-size": "3070", + "added-files-size": "3080", "added-records": "5", "changed-partition-count": "1", "iceberg-version": "Apache Iceberg Go " + iceberg.Version(), "total-data-files": "5", "total-delete-files": "0", "total-equality-deletes": "0", - "total-files-size": "3070", + "total-files-size": "3080", "total-position-deletes": "0", "total-records": "5", }, @@ -1182,16 +1182,16 @@ func (t *TableWritingTestSuite) TestReplaceDataFiles() { Operation: table.OpOverwrite, Properties: iceberg.Properties{ "added-data-files": "1", - "added-files-size": "963", + "added-files-size": "967", "added-records": "4", "deleted-data-files": "2", "deleted-records": "4", "iceberg-version": "Apache Iceberg Go " + iceberg.Version(), - "removed-files-size": "1816", + "removed-files-size": "1822", "total-data-files": "4", "total-delete-files": "0", "total-equality-deletes": "0", - "total-files-size": "3687", + "total-files-size": "3700", "total-position-deletes": "0", "total-records": "10", }, @@ -2565,7 +2565,7 @@ func arrowTableWithNull() arrow.Table { "float": 0.0, "double": 0.0, "time": "00:00:01.000000", - "timestamp": "2023-01-01T19:25:00.000000+08:00", + "timestamp": "2023-01-01T11:25:00.000000", "timestamptz": "2023-01-01T19:25:00.000000Z", "date": "2023-01-01", "uuid": "00000000-0000-0000-0000-000000000000", @@ -2603,7 +2603,7 @@ func arrowTableWithNull() arrow.Table { "float": 0.9, "double": 0.9, "time": "00:00:03.000000", - "timestamp": "2023-03-01T19:25:00.000000+08:00", + "timestamp": "2023-03-01T11:25:00.000000", "timestamptz": "2023-03-01T19:25:00.000000Z", "date": "2023-03-01", "uuid": "11111111-1111-1111-1111-111111111111", diff --git a/table/transaction_test.go b/table/transaction_test.go index 4bbfa7b8ec..a2af4ac0bf 100644 --- a/table/transaction_test.go +++ b/table/transaction_test.go @@ -220,7 +220,7 @@ func (s *SparkIntegrationTestSuite) TestDifferentDataTypes() { "long": 1, "float": 0.0, "double": 0.0, - "timestamp": "2023-01-01T19:25:00.000000+08:00", + "timestamp": "2023-01-01T11:25:00.000000", "timestamptz": "2023-01-01T19:25:00.000000Z", "date": "2023-01-01", "uuid": "00000000-0000-0000-0000-000000000000", @@ -258,7 +258,7 @@ func (s *SparkIntegrationTestSuite) TestDifferentDataTypes() { "long": 9, "float": 0.9, "double": 0.9, - "timestamp": "2023-03-01T19:25:00.000000+08:00", + "timestamp": "2023-03-01T11:25:00.000000", "timestamptz": "2023-03-01T19:25:00.000000Z", "date": "2023-03-01", "uuid": "11111111-1111-1111-1111-111111111111", diff --git a/table/variant_residual.go b/table/variant_residual.go index 3800c164c3..6e3a8e36f3 100644 --- a/table/variant_residual.go +++ b/table/variant_residual.go @@ -19,15 +19,18 @@ package table import ( "context" + "errors" "fmt" "log/slog" "strconv" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/bitutil" "github.com/apache/arrow-go/v18/arrow/compute" "github.com/apache/arrow-go/v18/arrow/extensions" "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet/variant" "github.com/apache/iceberg-go" "github.com/google/uuid" ) @@ -46,18 +49,15 @@ func augmentSchemaWithExtracts(fileSchema *iceberg.Schema, cols []iceberg.Varian return iceberg.NewSchemaWithIdentifiers(fileSchema.ID, fileSchema.IdentifierFieldIDs, fields...) } -// buildExtractColumn materializes one variant extract term into a typed Arrow array over rec. -func buildExtractColumn(col iceberg.VariantExtractColumn, rec arrow.RecordBatch, mem memory.Allocator) (arrow.Array, arrow.Field, error) { +// buildExtractColumn materializes one variant extract term into a typed Arrow array over rec, +// navigating columnarly via compute.VariantGet with a per-row fallback when the path can't be followed. +func buildExtractColumn(ctx context.Context, col iceberg.VariantExtractColumn, rec arrow.RecordBatch, mem memory.Allocator) (arrow.Array, arrow.Field, error) { typ := col.Term.Type().(iceberg.PrimitiveType) dt, err := TypeToArrowType(typ, false, false) if err != nil { return nil, arrow.Field{}, err } - bldr := array.NewBuilder(mem, dt) - defer bldr.Release() - - n := int(rec.NumRows()) varName := col.Term.Ref().Field().Name arr := resolveVariantSource(rec, col.Term.Ref().Field().ID, col.SourcePath) if arr == nil { @@ -68,7 +68,117 @@ func buildExtractColumn(col iceberg.VariantExtractColumn, rec arrow.RecordBatch, return nil, arrow.Field{}, fmt.Errorf("%w: variant extract column %q is not a VariantArray (got %T)", iceberg.ErrInvalidArgument, varName, arr) } - for i := range n { + out, err := extractColumnValues(ctx, varr, col, typ, dt, mem) + if err != nil { + return nil, arrow.Field{}, err + } + + field := arrow.Field{ + Name: col.Name, + Type: dt, + Nullable: true, + Metadata: arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{strconv.Itoa(col.FieldID)}), + } + + return out, field, nil +} + +// variantPathOf returns the extract term's member-name path (kept off the public BoundExtract interface); false if the term has none. +func variantPathOf(t iceberg.BoundExtract) (variant.VariantPath, bool) { + vp, ok := t.(interface { + VariantPath() variant.VariantPath + }) + if !ok { + return variant.VariantPath{}, false + } + + return vp.VariantPath(), true +} + +// extractColumnValues navigates columnarly then casts each leaf with iceberg's cast; a VariantGet +// navigation error (arrow-rs errors where the residual filter wants null-on-miss) falls back to per-row. +func extractColumnValues(ctx context.Context, varr *extensions.VariantArray, col iceberg.VariantExtractColumn, typ iceberg.PrimitiveType, dt arrow.DataType, mem memory.Allocator) (arrow.Array, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + path, ok := variantPathOf(col.Term) + if !ok { + return extractColumnValuesPerRow(ctx, varr, col, dt, mem) + } + if fast := tryShreddedTypedColumn(varr, path, dt, mem); fast != nil { + return fast, nil + } + if !varr.IsShredded() { + return extractColumnValuesPerRow(ctx, varr, col, dt, mem) + } + + extracted, err := compute.VariantGet(ctx, varr, compute.VariantGetOptions{Path: path}) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return nil, err + } + + return extractColumnValuesPerRow(ctx, varr, col, dt, mem) + } + defer extracted.Release() + + leaves, ok := extracted.(*extensions.VariantArray) + if !ok { + return extractColumnValuesPerRow(ctx, varr, col, dt, mem) + } + + bldr := array.NewBuilder(mem, dt) + defer bldr.Release() + + varName := col.Term.Ref().Field().Name + for i := range leaves.Len() { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } + if leaves.IsNull(i) { + bldr.AppendNull() + + continue + } + + leaf, verr := leaves.Value(i) + if verr != nil { + slog.Warn("variant extract: skipping undecodable variant value", "column", varName, "row", i, "err", verr) + bldr.AppendNull() + + continue + } + + lit, ok := iceberg.CastVariantLiteral(leaf, typ) + if !ok { + bldr.AppendNull() + + continue + } + + if aerr := appendExtractLiteral(bldr, lit); aerr != nil { + return nil, aerr + } + } + + return bldr.NewArray(), nil +} + +// extractColumnValuesPerRow is the row-by-row walk used when columnar navigation cannot follow +// the path; col.Term.ExtractValue navigates and casts, treating any path miss as null. +func extractColumnValuesPerRow(ctx context.Context, varr *extensions.VariantArray, col iceberg.VariantExtractColumn, dt arrow.DataType, mem memory.Allocator) (arrow.Array, error) { + bldr := array.NewBuilder(mem, dt) + defer bldr.Release() + + varName := col.Term.Ref().Field().Name + for i := range varr.Len() { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } if varr.IsNull(i) { bldr.AppendNull() @@ -91,18 +201,149 @@ func buildExtractColumn(col iceberg.VariantExtractColumn, rec arrow.RecordBatch, } if aerr := appendExtractLiteral(bldr, lit); aerr != nil { - return nil, arrow.Field{}, aerr + return nil, aerr } } - field := arrow.Field{ - Name: col.Name, - Type: dt, - Nullable: true, - Metadata: arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{strconv.Itoa(col.FieldID)}), + return bldr.NewArray(), nil +} + +// tryShreddedTypedColumn returns the field's typed leaf column when it is shredded to exactly dt, else nil. +func tryShreddedTypedColumn(varr *extensions.VariantArray, path variant.VariantPath, dt arrow.DataType, mem memory.Allocator) arrow.Array { + if path.Len() == 0 || varr.Data().Offset() != 0 { + return nil + } + tv := varr.Shredded() + if tv == nil || rootResidualHidesRows(varr, tv) { + return nil + } + n := varr.Len() + + var mask *memory.Buffer + shouldBail := false + mergeValidity := func(arr arrow.Array) { + if arr.Data().Offset() != 0 { + shouldBail = true + + return + } + if arr.NullN() == 0 { + return + } + if arr.Len() < n { + shouldBail = true + + return + } + vb := arr.Data().Buffers()[0] + if vb == nil { + shouldBail = true + + return + } + if mask == nil { + mask = memory.NewResizableBuffer(mem) + mask.Resize(int(bitutil.BytesForBits(int64(n)))) + copy(mask.Bytes(), vb.Bytes()) + + return + } + merged := bitutil.BitmapAndAlloc(mem, mask.Bytes(), vb.Bytes(), 0, 0, int64(n), 0) + mask.Release() + mask = merged + } + bail := func() arrow.Array { + if mask != nil { + mask.Release() + } + + return nil + } + + mergeValidity(varr.Storage()) + + cur := tv + for i := range path.Len() { + name, _, isField := path.StepAt(i) + if !isField { + return bail() + } + st, ok := cur.(*array.Struct) + if !ok { + return bail() + } + mergeValidity(st) + idx, ok := st.DataType().(*arrow.StructType).FieldIdx(name) + if !ok { + return bail() + } + field, ok := st.Field(idx).(*array.Struct) + if !ok { + return bail() + } + fty := field.DataType().(*arrow.StructType) + mergeValidity(field) // a null wrapper is an absent key (spec): fold it so the row reads null + if vIdx, ok := fty.FieldIdx("value"); ok { + if v := field.Field(vIdx); v.NullN() != v.Len() { + return bail() + } + } + tvIdx, ok := fty.FieldIdx("typed_value") + if !ok { + return bail() + } + cur = field.Field(tvIdx) + } + + if shouldBail || cur.Len() != n || !arrow.TypeEqual(cur.DataType(), dt) { + return bail() + } + + // no ancestor/row nulls: the leaf's own validity already describes the result, return it zero-copy + if mask == nil { + cur.Retain() + + return cur + } + + mergeValidity(cur) + if shouldBail { + return bail() + } + curData := cur.Data() + buffers := append([]*memory.Buffer(nil), curData.Buffers()...) + buffers[0] = mask + nullCount := n - bitutil.CountSetBits(mask.Bytes(), 0, n) + d := array.NewData(cur.DataType(), n, buffers, curData.Children(), nullCount, 0) + mask.Release() + out := array.MakeFromData(d) + d.Release() + + return out +} + +// rootResidualHidesRows reports whether any row's whole object lives in the root residual (value present, typed_value null) - not representable by the typed tree, so the fast path must fall back. +func rootResidualHidesRows(varr *extensions.VariantArray, tv arrow.Array) bool { + uv := varr.UntypedValues() + if uv == nil || tv.NullN() == 0 || uv.NullN() == uv.Len() { + return false + } + if tv.Data().Offset() != 0 || uv.Data().Offset() != 0 { + return true // non-zero child offset: can't safely bit-index; presume residual so the caller bails + } + tvb := tv.Data().Buffers()[0] + if tvb == nil { + return true // NullN()>0 with no validity buffer is non-canonical; bail conservatively + } + uvb := uv.Data().Buffers()[0] + for i := range varr.Len() { + valuePresent := uvb == nil || bitutil.BitIsSet(uvb.Bytes(), i) + if valuePresent && !bitutil.BitIsSet(tvb.Bytes(), i) { + return true + } } - return bldr.NewArray(), field, nil + return false } // resolveVariantSource locates the extract's source array by top-level field id @@ -263,7 +504,7 @@ func (as *arrowScan) extractResidualFilter(ctx context.Context, cols []iceberg.V derived := make([]arrow.Array, 0, len(cols)) fields := make([]arrow.Field, 0, len(cols)) for _, c := range cols { - arr, field, err := buildExtractColumn(c, rec, mem) + arr, field, err := buildExtractColumn(ctx, c, rec, mem) if err != nil { for _, a := range derived { a.Release() diff --git a/table/variant_residual_bench_test.go b/table/variant_residual_bench_test.go new file mode 100644 index 0000000000..97e185625c --- /dev/null +++ b/table/variant_residual_bench_test.go @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "context" + "fmt" + "strconv" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/extensions" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" +) + +func BenchmarkVariantExtract(b *testing.B) { + const n = 1 << 17 // ParquetBatchSizeDefault + mem := memory.DefaultAllocator + ctx := compute.WithAllocator(context.Background(), mem) + + flat := func(t arrow.DataType) *extensions.VariantType { + return shredStruct(arrow.Field{Name: "a", Type: t}) + } + nested := func(t arrow.DataType) *extensions.VariantType { + return shredStruct(arrow.Field{Name: "a", Type: arrow.StructOf(arrow.Field{Name: "b", Type: t})}) + } + + mkRows := func(shape, leaf string) []map[string]any { + val := func(i int) any { + if leaf == "string" { + return "v" + strconv.Itoa(i) + } + + return int64(i) + } + offType := func(i int) any { + if leaf == "string" { + return int64(i) + } + + return "x" + } + rows := make([]map[string]any, n) + for i := range rows { + switch { + case shape == "nested": + rows[i] = map[string]any{"a": map[string]any{"b": val(i)}} + case shape == "nulls" && i%100 == 0: + rows[i] = nil + case shape == "residual" && i%100 == 0: + rows[i] = map[string]any{"a": offType(i)} + default: + rows[i] = map[string]any{"a": val(i)} + } + } + + return rows + } + + leaves := []struct { + name string + dt arrow.DataType + typ iceberg.PrimitiveType + }{ + {"int64", arrow.PrimitiveTypes.Int64, iceberg.PrimitiveTypes.Int64}, + {"string", arrow.BinaryTypes.String, iceberg.PrimitiveTypes.String}, + } + + for _, leaf := range leaves { + for _, shape := range []string{"clean", "nulls", "residual", "unshredded", "nested"} { + var shred *extensions.VariantType + path := "$.a" + switch shape { + case "unshredded": + shred = nil + case "nested": + shred, path = nested(leaf.dt), "$.a.b" + default: + shred = flat(leaf.dt) + } + + rec, col := buildVariantExtractRec(b, mem, shred, path, leaf.typ, mkRows(shape, leaf.name)) + varr := resolveVariantSource(rec, col.Term.Ref().Field().ID, col.SourcePath).(*extensions.VariantArray) + typ := col.Term.Type().(iceberg.PrimitiveType) + dt, _ := TypeToArrowType(typ, false, false) + + mode := "fallback" + if ref := tryShreddedTypedColumn(varr, vpath(b, col.Term), dt, mem); ref != nil { + mode = "fastpath" + ref.Release() + } + + b.Run(fmt.Sprintf("%s/%s/%s/production", leaf.name, shape, mode), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + out, err := extractColumnValues(ctx, varr, col, typ, dt, mem) + if err != nil { + b.Fatal(err) + } + out.Release() + } + }) + b.Run(fmt.Sprintf("%s/%s/perrow", leaf.name, shape), func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + out, err := extractColumnValuesPerRow(ctx, varr, col, dt, mem) + if err != nil { + b.Fatal(err) + } + out.Release() + } + }) + + rec.Release() + } + } +} diff --git a/table/variant_residual_fastpath_test.go b/table/variant_residual_fastpath_test.go new file mode 100644 index 0000000000..2ac4c3b498 --- /dev/null +++ b/table/variant_residual_fastpath_test.go @@ -0,0 +1,633 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table + +import ( + "context" + "testing" + + "github.com/apache/arrow-go/v18/arrow" + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/compute" + "github.com/apache/arrow-go/v18/arrow/decimal128" + "github.com/apache/arrow-go/v18/arrow/extensions" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/arrow-go/v18/parquet/variant" + "github.com/apache/iceberg-go" + "github.com/stretchr/testify/require" +) + +// buildVariantExtractRec builds a "payload" variant column of shreddedType (nil => unshredded) from rows, plus the bound extract term for path/typ. +func buildVariantExtractRec(t testing.TB, mem memory.Allocator, shreddedType *extensions.VariantType, path string, typ iceberg.PrimitiveType, rows []map[string]any) (arrow.RecordBatch, iceberg.VariantExtractColumn) { + t.Helper() + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + + vt := shreddedType + if vt == nil { + vt = extensions.NewDefaultVariantType() + } + vb := extensions.NewVariantBuilder(mem, vt) + for _, row := range rows { + if row == nil { + vb.AppendNull() + + continue + } + var b variant.Builder + require.NoError(t, b.Append(row)) + v, err := b.Build() + require.NoError(t, err) + vb.Append(v) + } + pArr := vb.NewArray() + vb.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + arrSchema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: pArr.DataType(), Nullable: true, Metadata: md}}, nil) + rec := array.NewRecordBatch(arrSchema, []arrow.Array{pArr}, int64(pArr.Len())) + pArr.Release() + + term, err := iceberg.Extract("payload", path, typ).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + + return rec, col +} + +func shredStruct(fields ...arrow.Field) *extensions.VariantType { + return extensions.NewShreddedVariantType(arrow.StructOf(fields...)) +} + +// vpath fetches the extract term's member-name path (now off the public BoundExtract interface). +func vpath(tb testing.TB, term iceberg.BoundExtract) variant.VariantPath { + tb.Helper() + p, ok := variantPathOf(term) + require.True(tb, ok) + + return p +} + +// TestExtractFastPathParity: fast-path output must match the per-row walk on every branch; wantFast asserts whether it fires. +func TestExtractFastPathParity(t *testing.T) { + i64 := arrow.PrimitiveTypes.Int64 + + cases := []struct { + name string + shred *extensions.VariantType + path string + typ iceberg.PrimitiveType + rows []map[string]any + wantFast bool + }{ + { + name: "exact-match int64 shredded", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, {"a": int64(2)}, {"a": int64(3), "city": "x"}}, + wantFast: true, + }, + { + name: "exact-match with absent field yields null", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, {"b": int64(9)}, {"a": int64(3)}}, + wantFast: true, + }, + { + name: "nested exact-match int64 shredded", + shred: shredStruct(arrow.Field{Name: "a", Type: arrow.StructOf( + arrow.Field{Name: "b", Type: i64}, + )}), + path: "$.a.b", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{ + {"a": map[string]any{"b": int64(7)}}, + {"a": map[string]any{"b": int64(8)}}, + }, + wantFast: true, + }, + { + name: "promotion int32->int64 skips fast path", + shred: shredStruct(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int32}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int32(1)}, {"a": int32(2)}}, + wantFast: false, + }, + { + name: "int64 extracted as float64 skips fast path (iceberg nulls it)", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Float64, + rows: []map[string]any{{"a": int64(5)}}, + wantFast: false, + }, + { + name: "unshredded skips fast path", + shred: nil, + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, {"a": int64(2)}}, + wantFast: false, + }, + { + name: "field-level residual (mixed types) skips fast path", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, {"a": "not-an-int"}, {"a": int64(3)}}, + wantFast: false, + }, + { + name: "null row folds into validity", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, nil, {"a": int64(3)}}, + wantFast: true, + }, + { + name: "null row and absent field both null in merged mask", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, nil, {"b": int64(9)}, {"a": int64(4)}}, + wantFast: true, + }, + { + name: "nested absent intermediate folds into validity", + shred: shredStruct(arrow.Field{Name: "a", Type: arrow.StructOf( + arrow.Field{Name: "b", Type: i64}, + )}), + path: "$.a.b", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{ + {"a": map[string]any{"b": int64(7)}}, + {"b": int64(9)}, + {"a": map[string]any{"b": int64(3)}}, + }, + wantFast: true, + }, + { + name: "field absent from shredded schema skips fast path", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.c", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, {"a": int64(2)}}, + wantFast: false, + }, + { + name: "string clean shredded", + shred: shredStruct(arrow.Field{Name: "a", Type: arrow.BinaryTypes.String}), + path: "$.a", typ: iceberg.PrimitiveTypes.String, + rows: []map[string]any{{"a": "x"}, {"a": "yy"}, {"a": "zzz"}}, + wantFast: true, + }, + { + name: "string with null row folds into validity", + shred: shredStruct(arrow.Field{Name: "a", Type: arrow.BinaryTypes.String}), + path: "$.a", typ: iceberg.PrimitiveTypes.String, + rows: []map[string]any{{"a": "x"}, nil, {"a": "zzz"}}, + wantFast: true, + }, + { + name: "mask allocated then bail on field residual", + shred: shredStruct(arrow.Field{Name: "a", Type: i64}), + path: "$.a", typ: iceberg.PrimitiveTypes.Int64, + rows: []map[string]any{{"a": int64(1)}, nil, {"a": "str"}}, + wantFast: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + rec, col := buildVariantExtractRec(t, mem, tc.shred, tc.path, tc.typ, tc.rows) + defer rec.Release() + varr := resolveVariantSource(rec, col.Term.Ref().Field().ID, col.SourcePath).(*extensions.VariantArray) + dt, err := TypeToArrowType(tc.typ, false, false) + require.NoError(t, err) + + ref := tryShreddedTypedColumn(varr, vpath(t, col.Term), dt, mem) + if ref != nil { + defer ref.Release() + } + require.Equal(t, tc.wantFast, ref != nil, "fast path firing") + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + + want, err := extractColumnValuesPerRow(ctx, varr, col, dt, mem) + require.NoError(t, err) + defer want.Release() + + require.Truef(t, array.Equal(got, want), + "fast/columnar output diverges from per-row reference\n got=%v\nwant=%v", got, want) + + if ref != nil { + require.True(t, sharesDataBuffers(got, ref), "fast path must share the typed column's data (no per-row rebuild)") + } + }) + } +} + +// TestFastPathRootResidualObjectFallsBack: a row whose whole object is in the root residual (value present, typed_value null) must fall back, not be nulled. +func TestFastPathRootResidualObjectFallsBack(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + vt := extensions.NewShreddedVariantType(arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64})) + build := func(obj map[string]any) ([]byte, []byte) { + var b variant.Builder + require.NoError(t, b.Append(obj)) + v, err := b.Build() + require.NoError(t, err) + + return v.Metadata().Bytes(), v.Bytes() + } + m0, _ := build(map[string]any{"a": int64(1)}) + m1, v1 := build(map[string]any{"a": int64(5)}) + + storageType := vt.StorageType().(*arrow.StructType) + tvType := storageType.Field(2).Type.(*arrow.StructType) + aType := tvType.Field(0).Type.(*arrow.StructType) + + binArr := func(vals [][]byte) arrow.Array { + bb := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) + defer bb.Release() + for _, x := range vals { + if x == nil { + bb.AppendNull() + } else { + bb.Append(x) + } + } + + return bb.NewArray() + } + + i64b := array.NewInt64Builder(mem) + defer i64b.Release() + i64b.Append(1) + i64b.Append(0) + aTyped := i64b.NewArray() + defer aTyped.Release() + aVal := binArr([][]byte{nil, nil}) + defer aVal.Release() + aData := array.NewData(aType, 2, []*memory.Buffer{nil}, []arrow.ArrayData{aVal.Data(), aTyped.Data()}, 0, 0) + defer aData.Release() + aStruct := array.NewStructData(aData) + defer aStruct.Release() + + tvValidity := memory.NewBufferBytes([]byte{0x01}) + tvData := array.NewData(tvType, 2, []*memory.Buffer{tvValidity}, []arrow.ArrayData{aStruct.Data()}, 1, 0) + defer tvData.Release() + tvStruct := array.NewStructData(tvData) + defer tvStruct.Release() + + metaArr := binArr([][]byte{m0, m1}) + defer metaArr.Release() + rootVal := binArr([][]byte{nil, v1}) + defer rootVal.Release() + + storageData := array.NewData(storageType, 2, []*memory.Buffer{nil}, []arrow.ArrayData{metaArr.Data(), rootVal.Data(), tvStruct.Data()}, 0, 0) + defer storageData.Release() + storage := array.NewStructData(storageData) + defer storage.Release() + varr := array.NewExtensionArrayWithStorage(vt, storage).(*extensions.VariantArray) + defer varr.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + rec := array.NewRecordBatch(arrow.NewSchema([]arrow.Field{{Name: "payload", Type: vt, Nullable: true, Metadata: md}}, nil), []arrow.Array{varr}, 2) + defer rec.Release() + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + term, err := iceberg.Extract("payload", "$.a", iceberg.PrimitiveTypes.Int64).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + dt, err := TypeToArrowType(iceberg.PrimitiveTypes.Int64, false, false) + require.NoError(t, err) + + require.Nil(t, tryShreddedTypedColumn(varr, vpath(t, col.Term), dt, mem), "must not fast-path a root-residual object row") + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + want, err := extractColumnValuesPerRow(ctx, varr, col, dt, mem) + require.NoError(t, err) + defer want.Release() + + require.Truef(t, array.Equal(got, want), "got=%v want=%v", got, want) + require.False(t, got.IsNull(1), "root-residual row must be extracted, not nulled") + require.EqualValues(t, 5, got.(*array.Int64).Value(1)) +} + +// TestFastPathDecimalScaleNearMissFallsBack: a shredded decimal whose scale differs from the target (arrow.TypeEqual near-miss) must fall back, not return the mistyped column. +func TestFastPathDecimalScaleNearMissFallsBack(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + shredType := &arrow.Decimal128Type{Precision: 10, Scale: 2} + vt := extensions.NewShreddedVariantType(arrow.StructOf(arrow.Field{Name: "a", Type: shredType})) + + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(1)})) + v, err := b.Build() + require.NoError(t, err) + meta := v.Metadata().Bytes() + + sb := array.NewStructBuilder(mem, vt.StorageType().(*arrow.StructType)) + defer sb.Release() + metaB := sb.FieldBuilder(0).(*array.BinaryBuilder) + valB := sb.FieldBuilder(1).(*array.BinaryBuilder) + tvB := sb.FieldBuilder(2).(*array.StructBuilder) + aB := tvB.FieldBuilder(0).(*array.StructBuilder) + aValB := aB.FieldBuilder(0).(*array.BinaryBuilder) + aDecB := aB.FieldBuilder(1).(*array.Decimal128Builder) + + sb.Append(true) + metaB.Append(meta) + valB.AppendNull() + tvB.Append(true) + aB.Append(true) + aValB.AppendNull() + aDecB.Append(decimal128.FromI64(150)) // 1.50 at scale 2 + + storage := sb.NewStructArray() + defer storage.Release() + varr := array.NewExtensionArrayWithStorage(vt, storage).(*extensions.VariantArray) + defer varr.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + rec := array.NewRecordBatch(arrow.NewSchema([]arrow.Field{{Name: "payload", Type: vt, Nullable: true, Metadata: md}}, nil), []arrow.Array{varr}, 1) + defer rec.Release() + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + term, err := iceberg.Extract("payload", "$.a", iceberg.DecimalTypeOf(10, 4)).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + dt, err := TypeToArrowType(iceberg.DecimalTypeOf(10, 4), false, false) + require.NoError(t, err) + + require.Nil(t, tryShreddedTypedColumn(varr, vpath(t, col.Term), dt, mem), "scale near-miss must not fast-path") + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + want, err := extractColumnValuesPerRow(ctx, varr, col, dt, mem) + require.NoError(t, err) + defer want.Release() + require.Truef(t, array.Equal(got, want), "got=%v want=%v", got, want) +} + +// TestFastPathTimestampTzNearMissFallsBack: a shredded tz-aware timestamp extracted as a zoneless timestamp (arrow.TypeEqual near-miss on tz) must fall back. +func TestFastPathTimestampTzNearMissFallsBack(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + shredType := &arrow.TimestampType{Unit: arrow.Microsecond, TimeZone: "UTC"} + vt := extensions.NewShreddedVariantType(arrow.StructOf(arrow.Field{Name: "a", Type: shredType})) + + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(1)})) + v, err := b.Build() + require.NoError(t, err) + meta := v.Metadata().Bytes() + + sb := array.NewStructBuilder(mem, vt.StorageType().(*arrow.StructType)) + defer sb.Release() + metaB := sb.FieldBuilder(0).(*array.BinaryBuilder) + valB := sb.FieldBuilder(1).(*array.BinaryBuilder) + tvB := sb.FieldBuilder(2).(*array.StructBuilder) + aB := tvB.FieldBuilder(0).(*array.StructBuilder) + aValB := aB.FieldBuilder(0).(*array.BinaryBuilder) + aTsB := aB.FieldBuilder(1).(*array.TimestampBuilder) + + sb.Append(true) + metaB.Append(meta) + valB.AppendNull() + tvB.Append(true) + aB.Append(true) + aValB.AppendNull() + aTsB.Append(arrow.Timestamp(1_000_000)) + + storage := sb.NewStructArray() + defer storage.Release() + varr := array.NewExtensionArrayWithStorage(vt, storage).(*extensions.VariantArray) + defer varr.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + rec := array.NewRecordBatch(arrow.NewSchema([]arrow.Field{{Name: "payload", Type: vt, Nullable: true, Metadata: md}}, nil), []arrow.Array{varr}, 1) + defer rec.Release() + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + term, err := iceberg.Extract("payload", "$.a", iceberg.PrimitiveTypes.Timestamp).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + dt, err := TypeToArrowType(iceberg.PrimitiveTypes.Timestamp, false, false) + require.NoError(t, err) + + require.Nil(t, tryShreddedTypedColumn(varr, vpath(t, col.Term), dt, mem), "tz near-miss must not fast-path") + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + want, err := extractColumnValuesPerRow(ctx, varr, col, dt, mem) + require.NoError(t, err) + defer want.Release() + require.Truef(t, array.Equal(got, want), "got=%v want=%v", got, want) +} + +// sharesDataBuffers reports whether a and b share their non-validity buffers (validity may differ after a mask merge). +func sharesDataBuffers(a, b arrow.Array) bool { + ba, bb := a.Data().Buffers(), b.Data().Buffers() + if len(ba) != len(bb) || len(ba) < 2 { + return false + } + for i := 1; i < len(ba); i++ { + if ba[i] != bb[i] { + return false + } + } + + return true +} + +// TestFastPathWrapperFieldNullIsAbsentKey: a null per-key wrapper is an absent key (shredding spec), so the fast path reads null even if the typed_value child is left live by a non-conformant writer. +func TestFastPathWrapperFieldNullIsAbsentKey(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + vt := extensions.NewShreddedVariantType(arrow.StructOf(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64})) + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(1)})) + v, err := b.Build() + require.NoError(t, err) + meta := v.Metadata().Bytes() + + storageType := vt.StorageType().(*arrow.StructType) + tvType := storageType.Field(2).Type.(*arrow.StructType) // struct + aType := tvType.Field(0).Type.(*arrow.StructType) // a: struct + + binArr := func(vals [][]byte) arrow.Array { + bb := array.NewBinaryBuilder(mem, arrow.BinaryTypes.Binary) + defer bb.Release() + for _, x := range vals { + if x == nil { + bb.AppendNull() + } else { + bb.Append(x) + } + } + + return bb.NewArray() + } + + i64b := array.NewInt64Builder(mem) + defer i64b.Release() + i64b.Append(1) + i64b.Append(5) // row 1 typed_value stays live under the (about-to-be) null wrapper + aTyped := i64b.NewArray() + defer aTyped.Release() + aVal := binArr([][]byte{nil, nil}) + defer aVal.Release() + + // wrapper "a": row0 valid, row1 NULL (bit1=0), children left untouched -> non-cascaded shape + wrapValidity := memory.NewBufferBytes([]byte{0x01}) + aData := array.NewData(aType, 2, []*memory.Buffer{wrapValidity}, []arrow.ArrayData{aVal.Data(), aTyped.Data()}, 1, 0) + defer aData.Release() + aStruct := array.NewStructData(aData) + defer aStruct.Release() + + tvData := array.NewData(tvType, 2, []*memory.Buffer{nil}, []arrow.ArrayData{aStruct.Data()}, 0, 0) + defer tvData.Release() + tvStruct := array.NewStructData(tvData) + defer tvStruct.Release() + + metaArr := binArr([][]byte{meta, meta}) + defer metaArr.Release() + rootVal := binArr([][]byte{nil, nil}) + defer rootVal.Release() + + storageData := array.NewData(storageType, 2, []*memory.Buffer{nil}, []arrow.ArrayData{metaArr.Data(), rootVal.Data(), tvStruct.Data()}, 0, 0) + defer storageData.Release() + storage := array.NewStructData(storageData) + defer storage.Release() + varr := array.NewExtensionArrayWithStorage(vt, storage).(*extensions.VariantArray) + defer varr.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + rec := array.NewRecordBatch(arrow.NewSchema([]arrow.Field{{Name: "payload", Type: vt, Nullable: true, Metadata: md}}, nil), []arrow.Array{varr}, 2) + defer rec.Release() + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + term, err := iceberg.Extract("payload", "$.a", iceberg.PrimitiveTypes.Int64).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + require.Truef(t, got.IsNull(1), "null wrapper is an absent key (spec): must read null, got=%v", got) + require.EqualValues(t, 1, got.(*array.Int64).Value(0)) +} + +// TestFastPathSlicedOffsetFallsBack: a VariantArray sliced to a non-zero offset must bail from the fast path and still match the per-row reference. +func TestFastPathSlicedOffsetFallsBack(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + rec, col := buildVariantExtractRec(t, mem, shredStruct(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}), + "$.a", iceberg.PrimitiveTypes.Int64, []map[string]any{{"a": int64(1)}, {"a": int64(2)}, {"a": int64(3)}}) + defer rec.Release() + full := resolveVariantSource(rec, col.Term.Ref().Field().ID, col.SourcePath).(*extensions.VariantArray) + sliced := array.NewSlice(full, 1, int64(full.Len())).(*extensions.VariantArray) + defer sliced.Release() + dt, err := TypeToArrowType(iceberg.PrimitiveTypes.Int64, false, false) + require.NoError(t, err) + + require.Nil(t, tryShreddedTypedColumn(sliced, vpath(t, col.Term), dt, mem), "sliced (offset!=0) array must bail") + + got, err := extractColumnValues(ctx, sliced, col, iceberg.PrimitiveTypes.Int64, dt, mem) + require.NoError(t, err) + defer got.Release() + want, err := extractColumnValuesPerRow(ctx, sliced, col, dt, mem) + require.NoError(t, err) + defer want.Release() + require.Truef(t, array.Equal(got, want), "got=%v want=%v", got, want) +} + +// TestExtractColumnValuesContextCancelled: a cancelled context returns its error instead of a silent partial result. +func TestExtractColumnValuesContextCancelled(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + base := compute.WithAllocator(context.Background(), mem) + ctx, cancel := context.WithCancel(base) + cancel() + + rec, col := buildVariantExtractRec(t, mem, shredStruct(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}), + "$.a", iceberg.PrimitiveTypes.Int64, []map[string]any{{"a": int64(1)}}) + defer rec.Release() + + _, _, err := buildExtractColumn(ctx, col, rec, mem) + require.ErrorIs(t, err, context.Canceled) +} + +// TestExtractColumnValuesCancelDuringLoop: cancelling mid-scan stops the VariantGet loop instead of finishing it. +func TestExtractColumnValuesCancelDuringLoop(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + base := compute.WithAllocator(context.Background(), mem) + ctx, cancel := context.WithCancel(base) + + rows := make([]map[string]any, 1<<17) + for i := range rows { + rows[i] = map[string]any{"a": int32(i)} + } + rec, col := buildVariantExtractRec(t, mem, shredStruct(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int32}), + "$.a", iceberg.PrimitiveTypes.Int64, rows) + defer rec.Release() + + go cancel() + _, _, err := buildExtractColumn(ctx, col, rec, mem) + require.ErrorIs(t, err, context.Canceled) +} + +// TestMiddleTierColumnarCast drives the compute.VariantGet + iceberg-cast tier: an int32 leaf extracted as int64 bails the fast path and returns the promoted values. +func TestMiddleTierColumnarCast(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + rec, col := buildVariantExtractRec(t, mem, shredStruct(arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int32}), + "$.a", iceberg.PrimitiveTypes.Int64, []map[string]any{{"a": int32(1)}, {"a": int32(2)}}) + defer rec.Release() + varr := resolveVariantSource(rec, col.Term.Ref().Field().ID, col.SourcePath).(*extensions.VariantArray) + dt, err := TypeToArrowType(iceberg.PrimitiveTypes.Int64, false, false) + require.NoError(t, err) + + require.Nil(t, tryShreddedTypedColumn(varr, vpath(t, col.Term), dt, mem), "int32 leaf as int64 must not fast-path") + require.True(t, varr.IsShredded(), "still shredded, so extractColumnValues uses the compute.VariantGet tier") + + got, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer got.Release() + require.False(t, got.IsNull(0)) + require.EqualValues(t, 1, got.(*array.Int64).Value(0)) + require.EqualValues(t, 2, got.(*array.Int64).Value(1)) +} diff --git a/table/variant_shredded_test.go b/table/variant_shredded_test.go index b54f0e8194..babc68060b 100644 --- a/table/variant_shredded_test.go +++ b/table/variant_shredded_test.go @@ -86,7 +86,7 @@ func TestShreddedVariantTableScan(t *testing.T) { ext, ok := scanned.Schema().Field(0).Type.(arrow.ExtensionType) require.True(t, ok, "payload must be Variant extension type, got %T", scanned.Schema().Field(0).Type) - assert.Equal(t, "parquet.variant", ext.ExtensionName()) + assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName()) // Scan order is not guaranteed; check the set of "a" values is {0..nData-1}. col := scanned.Column(0).Data() diff --git a/table/variant_shredded_write_test.go b/table/variant_shredded_write_test.go index 4baf1a1e9f..d7157c5274 100644 --- a/table/variant_shredded_write_test.go +++ b/table/variant_shredded_write_test.go @@ -1323,7 +1323,10 @@ func TestShreddedVariantExtractResidualNoLeak(t *testing.T) { require.NoError(t, err) defer f.Close() - tbl, err := pqarrow.ReadTable(context.Background(), f, nil, pqarrow.ArrowReadProperties{}, memory.DefaultAllocator) + // read rec on the checked allocator so the fast-path zero-copy leaf is leak-tracked + checked := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer checked.AssertSize(t, 0) + tbl, err := pqarrow.ReadTable(context.Background(), f, nil, pqarrow.ArrowReadProperties{}, checked) require.NoError(t, err) defer tbl.Release() @@ -1342,7 +1345,6 @@ func TestShreddedVariantExtractResidualNoLeak(t *testing.T) { bound, err := iceberg.BindExpr(fileSchema, pred, true) require.NoError(t, err) - checked := memory.NewCheckedAllocator(memory.DefaultAllocator) ctx := compute.WithAllocator(context.Background(), checked) as := &arrowScan{boundRowFilter: bound, caseSensitive: true} @@ -1352,9 +1354,7 @@ func TestShreddedVariantExtractResidualNoLeak(t *testing.T) { out, err := fn(rec) require.NoError(t, err) - out.Release() - - checked.AssertSize(t, 0) + defer out.Release() } // TestVariantExtractResidualCombined exercises AND/OR filters over multiple extract terms. @@ -1929,7 +1929,7 @@ func TestBuildExtractColumnRenamedSource(t *testing.T) { require.NoError(t, err) col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"old_payload"}} - arr, _, err := buildExtractColumn(col, rec, mem) + arr, _, err := buildExtractColumn(context.Background(), col, rec, mem) require.NoError(t, err) defer arr.Release() require.Equal(t, 1, arr.Len()) @@ -1962,7 +1962,7 @@ func TestBuildExtractColumnResolvesByFieldID(t *testing.T) { // SourcePath deliberately mismatches the physical name, so only the field-id match can resolve it. col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} - arr, _, err := buildExtractColumn(col, rec, mem) + arr, _, err := buildExtractColumn(context.Background(), col, rec, mem) require.NoError(t, err) defer arr.Release() require.Equal(t, 1, arr.Len()) @@ -1994,7 +1994,7 @@ func TestBuildExtractColumnDottedName(t *testing.T) { require.NoError(t, err) col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"a.b"}} - arr, _, err := buildExtractColumn(col, rec, mem) + arr, _, err := buildExtractColumn(context.Background(), col, rec, mem) require.NoError(t, err) defer arr.Release() require.Equal(t, 1, arr.Len()) @@ -2040,7 +2040,7 @@ func TestBuildExtractColumnNested(t *testing.T) { require.NoError(t, err) col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"wrapper", "payload"}} - arr, _, err := buildExtractColumn(col, rec, mem) + arr, _, err := buildExtractColumn(context.Background(), col, rec, mem) require.NoError(t, err) defer arr.Release() require.Equal(t, 1, arr.Len()) @@ -2068,10 +2068,124 @@ func TestBuildExtractColumnWrongType(t *testing.T) { require.NoError(t, err) col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x"} - _, _, err = buildExtractColumn(col, rec, mem) + _, _, err = buildExtractColumn(context.Background(), col, rec, mem) require.ErrorIs(t, err, iceberg.ErrInvalidArgument) } +// TestBuildExtractColumnShreddedColumnar exercises the compute.VariantGet fast path over a +// shredded column and asserts per-row values match. +func TestBuildExtractColumnShreddedColumnar(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + + shredded := extensions.NewShreddedVariantType(arrow.StructOf( + arrow.Field{Name: "a", Type: arrow.PrimitiveTypes.Int64}, + )) + vb := extensions.NewVariantBuilder(mem, shredded) + for i := range 4 { + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(i * 10), "city": "NYC"})) + v, err := b.Build() + require.NoError(t, err) + vb.Append(v) + } + pArr := vb.NewArray() + vb.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + arrSchema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: pArr.DataType(), Nullable: true, Metadata: md}}, nil) + rec := array.NewRecordBatch(arrSchema, []arrow.Array{pArr}, int64(pArr.Len())) + pArr.Release() + defer rec.Release() + + term, err := iceberg.Extract("payload", "$.a", iceberg.PrimitiveTypes.Int64).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + + arr, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer arr.Release() + require.Equal(t, 4, arr.Len()) + got := arr.(*array.Int64) + for i := range 4 { + require.EqualValues(t, i*10, got.Value(i), "row %d", i) + } +} + +// TestBuildExtractColumnKeepsIcebergCast guards that the columnar path casts with iceberg's +// restrictive CastVariantLiteral, not arrow-go's permissive cast: an int64 extracted as float64 +// is null under iceberg's cast (no int->float coercion), whereas arrow-go's cast would yield 5.0. +func TestBuildExtractColumnKeepsIcebergCast(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + + vb := extensions.NewVariantBuilder(mem, extensions.NewDefaultVariantType()) + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(5)})) + v, err := b.Build() + require.NoError(t, err) + vb.Append(v) + pArr := vb.NewArray() + vb.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + arrSchema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: pArr.DataType(), Nullable: true, Metadata: md}}, nil) + rec := array.NewRecordBatch(arrSchema, []arrow.Array{pArr}, 1) + pArr.Release() + defer rec.Release() + + term, err := iceberg.Extract("payload", "$.a", iceberg.PrimitiveTypes.Float64).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + + arr, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer arr.Release() + require.Equal(t, 1, arr.Len()) + require.True(t, arr.IsNull(0), "int64 must not cast to float64 under iceberg's cast") +} + +// TestBuildExtractColumnFieldIntoScalarFallsBack: "$.a.b" steps into a scalar, so compute.VariantGet +// errors and buildExtractColumn falls back to the per-row walk, which yields null (no error). +func TestBuildExtractColumnFieldIntoScalarFallsBack(t *testing.T) { + mem := memory.NewCheckedAllocator(memory.DefaultAllocator) + defer mem.AssertSize(t, 0) + ctx := compute.WithAllocator(t.Context(), mem) + + iceSchema := iceberg.NewSchema(0, iceberg.NestedField{ID: 2, Name: "payload", Type: iceberg.VariantType{}}) + + vb := extensions.NewVariantBuilder(mem, extensions.NewDefaultVariantType()) + var b variant.Builder + require.NoError(t, b.Append(map[string]any{"a": int64(5)})) + v, err := b.Build() + require.NoError(t, err) + vb.Append(v) + pArr := vb.NewArray() + vb.Release() + + md := arrow.NewMetadata([]string{ArrowParquetFieldIDKey}, []string{"2"}) + arrSchema := arrow.NewSchema([]arrow.Field{{Name: "payload", Type: pArr.DataType(), Nullable: true, Metadata: md}}, nil) + rec := array.NewRecordBatch(arrSchema, []arrow.Array{pArr}, 1) + pArr.Release() + defer rec.Release() + + term, err := iceberg.Extract("payload", "$.a.b", iceberg.PrimitiveTypes.Int64).Bind(iceSchema, true) + require.NoError(t, err) + col := iceberg.VariantExtractColumn{Term: term.(iceberg.BoundExtract), FieldID: 100, Name: "_x", SourcePath: []string{"payload"}} + + arr, _, err := buildExtractColumn(ctx, col, rec, mem) + require.NoError(t, err) + defer arr.Release() + require.Equal(t, 1, arr.Len()) + require.True(t, arr.IsNull(0), "field step into a scalar yields null via the per-row fallback") +} + // TestInclusiveProjectionExtractAlwaysTrue covers the manifest-eval projection path // for an extract term: variant can't be a partition source, so FieldsBySourceID is // empty and the projected partition predicate is AlwaysTrue (no false pruning). diff --git a/variant_extract.go b/variant_extract.go index e9e826331b..6877906dd6 100644 --- a/variant_extract.go +++ b/variant_extract.go @@ -159,6 +159,16 @@ func (b *boundExtract[T]) Ref() BoundReference { return b.ref } func (b *boundExtract[T]) Type() Type { return b.typ } func (b *boundExtract[T]) Path() string { return b.path } +// VariantPath returns the term's member-name path for columnar extraction via compute.VariantGet. +func (b *boundExtract[T]) VariantPath() variant.VariantPath { + var p variant.VariantPath + for _, f := range b.fields { + p = p.Field(f) + } + + return p +} + // MarshalJSON reports that extract terms have no REST expression-JSON form (see ErrExtractNotSerializable), rather than silently encoding to {}. func (*boundExtract[T]) MarshalJSON() ([]byte, error) { return nil, ErrExtractNotSerializable }