Skip to content
10 changes: 5 additions & 5 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
github.com/RoaringBitmap/roaring/v2 v2.26.0
github.com/alexflint/go-arg v1.6.1
github.com/apache/arrow-go/v18 v18.7.0
github.com/apache/arrow-go/v18 v18.8.0
github.com/aws/aws-sdk-go-v2 v1.45.1
github.com/aws/aws-sdk-go-v2/config v1.33.1
github.com/aws/aws-sdk-go-v2/credentials v1.20.1
Expand Down Expand Up @@ -95,7 +95,7 @@ require (
github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/alexflint/go-scalar v1.2.0 // indirect
github.com/andybalholm/brotli v1.2.2 // indirect
github.com/andybalholm/brotli v1.2.3 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apache/thrift v0.24.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect
Expand Down Expand Up @@ -184,7 +184,7 @@ require (
github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect
github.com/magiconair/properties v1.8.10 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/mattn/go-shellwords v1.0.13 // indirect
github.com/mattn/go-sqlite3 v1.14.34 // indirect
Expand Down Expand Up @@ -276,9 +276,9 @@ require (
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gotest.tools/v3 v3.5.2 // indirect
modernc.org/libc v1.73.4 // indirect
modernc.org/libc v1.74.4 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.11.0 // indirect
modernc.org/sqlite v1.53.0 // indirect
modernc.org/sqlite v1.57.0 // indirect
tags.cncf.io/container-device-interface v1.1.0 // indirect
)
37 changes: 18 additions & 19 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@ github.com/alexflint/go-scalar v1.2.0 h1:WR7JPKkeNpnYIOfHRa7ivM21aWAdHD0gEWHCx+W
github.com/alexflint/go-scalar v1.2.0/go.mod h1:LoFvNMqS1CPrMVltza4LvnGKhaSpc3oyLEBUZVhhS2o=
github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30=
github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM=
github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/andybalholm/brotli v1.2.3 h1:8H1qwOkl2LPfjf3YezB90JnCliZb6SInJ/OJkEbA5NQ=
github.com/andybalholm/brotli v1.2.3/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ=
github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw=
github.com/apache/arrow-go/v18 v18.7.0 h1:Vw/i+cJyebUofT7JlqFpe65LrmwxULn166jjwStM4HY=
github.com/apache/arrow-go/v18 v18.7.0/go.mod h1:PM6IigLJkdMwIpeHXnymo+xZ52f42a9EYiLtRel4p/A=
github.com/apache/arrow-go/v18 v18.8.0 h1:BLOzbPv7bxMPgXPacAg6HQjnxupYsZzC4tf+FkqPU/M=
github.com/apache/arrow-go/v18 v18.8.0/go.mod h1:uJCFfCwq0KsxCmsCfQg4ft+LsW+iHYzAXiSDh5ug/8U=
github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ=
github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs=
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so=
Expand Down Expand Up @@ -363,8 +363,8 @@ github.com/google/go-replayers/httpreplay v1.2.0 h1:VM1wEyyjaoU53BwrOnaf9VhAyQQE
github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy+tME4bwyqPcwWbNlUI1Mcg=
github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc=
github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg=
github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0=
github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
Expand Down Expand Up @@ -440,8 +440,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU=
github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
Expand Down Expand Up @@ -766,7 +766,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down Expand Up @@ -824,20 +823,20 @@ gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc=
k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0=
modernc.org/cc/v4 v4.28.4 h1:Hd/4Es+MBj+/7hSdZaisNyu6bv3V0Dp2MdllyfqaH+c=
modernc.org/cc/v4 v4.28.4/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.4 h1:OVnSOWQjVKOYkFxoHYB+qQmSHK5gqMqARM+K9DpR/Ws=
modernc.org/ccgo/v4 v4.34.4/go.mod h1:qdKqE8FNIYyysougB1RX9MxCzp5oJOcQXSobANJ4TuE=
modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI=
modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU=
modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.3 h1:6QAplYyVO+KdPW3pGnqmJDUxtkec8ooEWvks/hhU3lc=
modernc.org/gc/v3 v3.1.3/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI=
modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.73.4 h1:+ra4Ui8ngyt8HDcO1FTDPWlkAh6yOdaO2yAoh8MddQA=
modernc.org/libc v1.73.4/go.mod h1:DXZ3eO8qMCNn2SnmTNCiC71nJ9Rcq3PsnpU6Vc4rWK8=
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
Expand All @@ -846,8 +845,8 @@ modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.53.0 h1:20WG8N9q4ji/dEqGk4uiI0c6OPjSeLTNYGFCc3+7c1M=
modernc.org/sqlite v1.53.0/go.mod h1:xoEpOIpGrgT48H5iiyt/YXPCZPEzlfmfFwtk8Lklw8s=
modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg=
modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
Expand Down
2 changes: 1 addition & 1 deletion table/arrow_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,7 +436,7 @@ func (c convertToIceberg) Primitive(dt arrow.DataType) (result iceberg.NestedFie
switch dt.ExtensionName() {
case "arrow.uuid":
result.Type = iceberg.PrimitiveTypes.UUID
case "parquet.variant":
case extensions.VariantExtensionName, extensions.LegacyVariantExtensionName:
Comment thread
nssalian marked this conversation as resolved.
Resolved
Comment thread
nssalian marked this conversation as resolved.
Resolved
result.Type = iceberg.VariantType{}
case "geoarrow.wkb":
wkb, ok := dt.(*geoarrow.WKBType)
Expand Down
24 changes: 12 additions & 12 deletions table/arrow_utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,7 +389,7 @@ func TestVariantArrowConversion(t *testing.T) {
dataField := arrowSc.Field(1)
ext, ok := dataField.Type.(arrow.ExtensionType)
require.True(t, ok, "expected extension type, got %T", dataField.Type)
assert.Equal(t, "parquet.variant", ext.ExtensionName())
assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName())

ice, err := table.ArrowSchemaToIceberg(arrowSc, false, nil)
require.NoError(t, err)
Expand Down Expand Up @@ -427,7 +427,7 @@ func TestVariantArrowConversion(t *testing.T) {
for _, idx := range []int{1, 2} {
ext, ok := arrowSc.Field(idx).Type.(arrow.ExtensionType)
require.True(t, ok, "field %d should be extension type", idx)
assert.Equal(t, "parquet.variant", ext.ExtensionName())
assert.Equal(t, extensions.VariantExtensionName, ext.ExtensionName())
}

ice, err := table.ArrowSchemaToIceberg(arrowSc, false, nil)
Expand Down Expand Up @@ -1199,28 +1199,28 @@ func ArrowRecordWithAllTimestampPrec(mem memory.Allocator) arrow.RecordBatch {
batch, _, err := array.RecordFromJSON(mem, ArrowSchemaWithAllTimestampPrec,
strings.NewReader(`[
{
"timestamp_s": "2023-01-01T19:25:00-05:00",
"timestamp_s": "2023-01-01T19:25:00",
"timestamptz_s": "2023-01-01T19:25:00Z",
"timestamp_ms": "2023-01-01T19:25:00.123-05:00",
"timestamp_ms": "2023-01-01T19:25:00.123",
"timestamptz_ms": "2023-01-01T19:25:00.123Z",
"timestamp_us": "2023-01-01T19:25:00.123456-05:00",
"timestamp_us": "2023-01-01T19:25:00.123456",
"timestamptz_us": "2023-01-01T19:25:00.123456Z",
"timestamp_ns": "2024-07-11T03:30:00.123456789-05:00",
"timestamp_ns": "2024-07-11T03:30:00.123456789",
"timestamptz_ns": "2023-01-01T19:25:00.123456789Z",
"timestamptz_us_etc_utc": "2023-01-01T19:25:00.123456Z",
"timestamptz_ns_z": "2024-07-11T03:30:00.123456789Z",
"timestamptz_s_0000": "2023-01-01T19:25:00Z"
}, {}, {
"timestamp_s": "2023-03-01T19:25:00-05:00",
"timestamp_s": "2023-03-01T19:25:00",
"timestamptz_s": "2023-03-01T19:25:00Z",
"timestamp_ms": "2023-03-01T19:25:00.123-05:00",
"timestamp_ms": "2023-03-01T19:25:00.123",
"timestamptz_ms": "2023-03-01T19:25:00.123Z",
"timestamp_us": "2023-03-01T19:25:00.123456-05:00",
"timestamp_us": "2023-03-01T19:25:00.123456",
"timestamptz_us": "2023-03-01T19:25:00.123456Z",
"timestamp_ns": "2024-07-11T03:30:00.9876543210-05:00",
"timestamptz_ns": "2023-03-01T19:25:00.9876543210Z",
"timestamp_ns": "2024-07-11T03:30:00.987654321",
"timestamptz_ns": "2023-03-01T19:25:00.987654321Z",
"timestamptz_us_etc_utc": "2023-03-01T19:25:00.123456Z",
"timestamptz_ns_z": "2024-07-11T03:30:00.9876543210Z",
"timestamptz_ns_z": "2024-07-11T03:30:00.987654321Z",
"timestamptz_s_0000": "2023-03-01T19:25:00Z"
}
]`))
Expand Down
10 changes: 5 additions & 5 deletions table/equality_delete_reader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -811,11 +811,11 @@ func TestEqualityDeleteNullableFastPathKeys(t *testing.T) {
keyType: iceberg.PrimitiveTypes.Timestamp,
dataJSON: `[
{"row_id": 1, "key": null},
{"row_id": 2, "key": "1970-01-01T00:00:00.000000Z"},
{"row_id": 3, "key": "2024-01-02T03:04:05.000000Z"}
{"row_id": 2, "key": "1970-01-01T00:00:00.000000"},
{"row_id": 3, "key": "2024-01-02T03:04:05.000000"}
]`,
nullDeleteJSON: `[{"key": null}]`,
zeroDeleteJSON: `[{"key": "1970-01-01T00:00:00.000000Z"}]`,
zeroDeleteJSON: `[{"key": "1970-01-01T00:00:00.000000"}]`,
},
}

Expand Down Expand Up @@ -880,7 +880,7 @@ func TestEqualityDeleteNullableCompositeFastPathKey(t *testing.T) {
name: "partially null key",
dataJSON: `[
{"row_id": 1, "id": 7, "deleted_at": null},
{"row_id": 2, "id": 7, "deleted_at": "1970-01-01T00:00:00.000000Z"},
{"row_id": 2, "id": 7, "deleted_at": "1970-01-01T00:00:00.000000"},
{"row_id": 3, "id": 8, "deleted_at": null},
{"row_id": 4, "id": null, "deleted_at": null}
]`,
Expand All @@ -891,7 +891,7 @@ func TestEqualityDeleteNullableCompositeFastPathKey(t *testing.T) {
name: "all-null key",
dataJSON: `[
{"row_id": 1, "id": null, "deleted_at": null},
{"row_id": 2, "id": null, "deleted_at": "1970-01-01T00:00:00.000000Z"},
{"row_id": 2, "id": null, "deleted_at": "1970-01-01T00:00:00.000000"},
{"row_id": 3, "id": 7, "deleted_at": null}
]`,
deleteJSON: `[{"id": null, "deleted_at": null}]`,
Expand Down
106 changes: 102 additions & 4 deletions table/internal/parquet_files.go
Original file line number Diff line number Diff line change
Expand Up @@ -613,13 +613,13 @@ func (p parquetFormat) NewFileWriter(ctx context.Context, fs iceio.WriteFileIO,

counter := &internal.CountingWriter{W: fw}
mem := compute.GetAllocator(ctx)
writerProps, err := getWriteProperties(info.WriteProps, arrowSchema)
arrProps := writeArrowProps(mem)
writerProps, err := getWriteProperties(info.WriteProps, arrowSchema, arrProps)
if err != nil {
fw.Close()

return nil, err
}
arrProps := pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithStoreSchema())

writer, err := pqarrow.NewFileWriter(arrowSchema, counter, writerProps, arrProps)
if err != nil {
Expand Down Expand Up @@ -691,9 +691,13 @@ func typeHasDecimal128(dt arrow.DataType) bool {
return false
}

func writeArrowProps(mem memory.Allocator) pqarrow.ArrowWriterProperties {
return pqarrow.NewArrowWriterProperties(pqarrow.WithAllocator(mem), pqarrow.WithStoreSchema())
}

// getWriteProperties requires explicit write properties so misconfigured writer
// plumbing fails fast instead of silently defaulting Parquet settings.
func getWriteProperties(writeProps any, arrowSchema *arrow.Schema) (*parquet.WriterProperties, error) {
func getWriteProperties(writeProps any, arrowSchema *arrow.Schema, arrProps pqarrow.ArrowWriterProperties) (*parquet.WriterProperties, error) {
if writeProps == nil {
return nil, fmt.Errorf("%w: write properties are required", iceberg.ErrInvalidArgument)
}
Expand All @@ -711,9 +715,103 @@ func getWriteProperties(writeProps any, arrowSchema *arrow.Schema) (*parquet.Wri
wp = append(wp, parquet.WithStoreDecimalAsInteger(true))
}

// Match Iceberg Java: apply parquet-mr's cost-based dictionary fallback to every leaf
// column so high-cardinality columns fall back to PLAIN rather than keeping a dictionary.
// arrow-go otherwise enables it only for uncompressed columns, so zstd (our default) would
// retain dictionaries on all-distinct columns and roughly double their size.
costFallback, err := dictCostFallbackProps(arrowSchema, wp, arrProps)
if err != nil {
return nil, err
}
wp = append(wp, costFallback...)

return parquet.NewWriterProperties(wp...), nil
}

// dictCostFallbackProps returns a WithDictionaryCostFallbackFor(true) property per leaf, walking the arrow schema directly (extensions unwrapped) and falling back to pqarrow.ToParquet for list/map schemas.
func dictCostFallbackProps(arrowSchema *arrow.Schema, base []parquet.WriterProperty, arrProps pqarrow.ArrowWriterProperties) ([]parquet.WriterProperty, error) {
if schemaHasListOrMap(arrowSchema) {
return dictCostFallbackViaParquet(arrowSchema, base, arrProps)
}

var props []parquet.WriterProperty
var walk func(prefix string, dt arrow.DataType)
walk = func(prefix string, dt arrow.DataType) {
if ext, ok := dt.(arrow.ExtensionType); ok {
dt = ext.StorageType()
}
if d, ok := dt.(*arrow.DictionaryType); ok {
dt = d.ValueType
}
if r, ok := dt.(*arrow.RunEndEncodedType); ok {
dt = r.Encoded()
}
if st, ok := dt.(*arrow.StructType); ok {
for _, f := range st.Fields() {
walk(prefix+"."+f.Name, f.Type)
}

return
}
props = append(props, parquet.WithDictionaryCostFallbackFor(prefix, true))
}
for _, f := range arrowSchema.Fields() {
walk(f.Name, f.Type)
}

return props, nil
}

// dictCostFallbackViaParquet is the authoritative path for list/map schemas, whose parquet leaf naming the direct walk does not reproduce.
func dictCostFallbackViaParquet(arrowSchema *arrow.Schema, base []parquet.WriterProperty, arrProps pqarrow.ArrowWriterProperties) ([]parquet.WriterProperty, error) {
parquetSchema, err := pqarrow.ToParquet(arrowSchema, parquet.NewWriterProperties(base...), arrProps)
if err != nil {
return nil, err
}

props := make([]parquet.WriterProperty, 0, parquetSchema.NumColumns())
for i := range parquetSchema.NumColumns() {
props = append(props, parquet.WithDictionaryCostFallbackFor(parquetSchema.Column(i).Path(), true))
}

return props, nil
}

func schemaHasListOrMap(sc *arrow.Schema) bool {
var has func(dt arrow.DataType) bool
has = func(dt arrow.DataType) bool {
if ext, ok := dt.(arrow.ExtensionType); ok {
dt = ext.StorageType()
}
if d, ok := dt.(*arrow.DictionaryType); ok {
dt = d.ValueType
}
if r, ok := dt.(*arrow.RunEndEncodedType); ok {
dt = r.Encoded()
}
switch t := dt.(type) {
case *arrow.ListType, *arrow.LargeListType, *arrow.FixedSizeListType,
Comment thread
nssalian marked this conversation as resolved.
Resolved
*arrow.ListViewType, *arrow.LargeListViewType, *arrow.MapType:
return true
case *arrow.StructType:
for _, f := range t.Fields() {
if has(f.Type) {
return true
}
}
}

return false
}
for _, f := range sc.Fields() {
if has(f.Type) {
return true
}
}

return false
}

// Write appends a record batch to the Parquet file.
func (w *ParquetFileWriter) Write(batch arrow.RecordBatch) error {
writeBatch, err := w.normalizeGeoBatch(batch)
Expand Down Expand Up @@ -2371,7 +2469,7 @@ func (p *pruneParquetSchema) Field(field pqarrow.SchemaField, result arrow.Field

// Variant is an extension type wrapping a struct (metadata + value).
// Select the entire field including all children.
if ext, ok := field.Field.Type.(arrow.ExtensionType); ok && ext.ExtensionName() == "parquet.variant" {
if ext, ok := field.Field.Type.(arrow.ExtensionType); ok && extensions.IsVariantExtensionName(ext.ExtensionName()) {
return p.projectVariant(field)
}

Expand Down
Loading
Loading