Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions table/arrow_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -2345,7 +2345,7 @@ func unpartitionedWrite(ctx context.Context, factory *writerFactory, records ite
defer close(outputCh)
defer factory.stopCount()

writer := factory.newRollingDataWriter(writerCtx, "", nil, outputCh)
writer := factory.newRollingDataWriter(writerCtx, "", nil, outputCh, factory.currentSpec)
for rec, err := range records {
if err != nil {
errCh <- err
Expand Down Expand Up @@ -2444,7 +2444,18 @@ func positionDeleteRecordsToDataFiles(ctx context.Context, rootLocation string,
targetFileSize := int64(meta.props.GetInt(WriteDeleteTargetFileSizeBytesKey,
WriteDeleteTargetFileSizeBytesDefault))

if latestMetadata.PartitionSpec().IsUnpartitioned() {
currentSpec := latestMetadata.PartitionSpec()
// The unpartitioned writer stamps the current spec on every file, so it is
// safe only when every target file was written under that spec.
useUnpartitionedWriter := currentSpec.IsUnpartitioned()
for _, partition := range partitionContextByFilePath {
if int(partition.specID) != currentSpec.ID() {
useUnpartitionedWriter = false

break
}
}
Comment on lines +2447 to +2457

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: a short comment on why would help here: the unpartitioned writer stamps the current spec on every file, so it is safe only when every target-file was written under that spec

without it, someone may simplify this back to "IsUnpartitioned()"

if useUnpartitionedWriter {
cw := newConcurrentDataFileWriter(func(rootLocation string, fs iceio.WriteFileIO, meta *MetadataBuilder, props iceberg.Properties, opts ...dataFileWriterOption) (dataFileWriter, error) {
return newPositionDeleteWriter(rootLocation, fs, meta, props, opts...)
}, withSchemaSanitization(false))
Expand Down
2 changes: 1 addition & 1 deletion table/clustered_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ func clusteredPartitionedWrite(
}
partitionPath := extractionPlan.pathPlan.format(part.partitionRec)
currentWriter = factory.newRollingDataWriter(
ctx, partitionPath, part.partitionValues, outputCh)
ctx, partitionPath, part.partitionValues, outputCh, factory.currentSpec)
currentRec = part.partitionRec
}

Expand Down
2 changes: 1 addition & 1 deletion table/delete_parquet_settings_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestDeleteParquetSettingsOnDisk(t *testing.T) {
defer factory.stopCount()
writerProps = factory.writeProps.([]parquet.WriterProperty)
rowGroupBytes = factory.rowGroupBytes
writer, err := factory.openFileWriter(ctx, "", nil, 0, 1, nil)
writer, err := factory.openFileWriter(ctx, "", nil, 0, 1, nil, factory.currentSpec)
require.NoError(t, err)
for _, rec := range batches {
require.NoError(t, writer.Write(rec))
Expand Down
2 changes: 1 addition & 1 deletion table/parquet_bloom_properties_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func TestParquetBloomLogicalPathsOnDisk(t *testing.T) {
}, builder, fileSchema, 1048576, withFactoryFileSchema(fileSchema))
require.NoError(t, err)
defer factory.stopCount()
writer, err := factory.openFileWriter(t.Context(), "", nil, 0, 1, nil)
writer, err := factory.openFileWriter(t.Context(), "", nil, 0, 1, nil, factory.currentSpec)
require.NoError(t, err)
require.NoError(t, writer.Write(batch))
df, err = writer.Close()
Expand Down
4 changes: 2 additions & 2 deletions table/parquet_column_stats_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ func writeStatsSettingsFile(t *testing.T, mode string, props iceberg.Properties)
require.NoError(t, err)
defer factory.closeAll()
output := make(chan iceberg.DataFile, 1)
writer := factory.newRollingDataWriter(t.Context(), "", nil, output)
writer := factory.newRollingDataWriter(t.Context(), "", nil, output, factory.currentSpec)
require.NoError(t, writer.Add(record))
record.Release()
require.NoError(t, writer.closeAndWait())
Expand Down Expand Up @@ -267,7 +267,7 @@ func writeStatsSettingsDeleteFile(t *testing.T, mode string, content iceberg.Man
require.NoError(t, err)
defer factory.closeAll()
output := make(chan iceberg.DataFile, 1)
writer := factory.newRollingDataWriter(t.Context(), "", nil, output)
writer := factory.newRollingDataWriter(t.Context(), "", nil, output, factory.currentSpec)
require.NoError(t, writer.Add(record))
record.Release()
require.NoError(t, writer.closeAndWait())
Expand Down
2 changes: 1 addition & 1 deletion table/partitioned_fanout_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ func (p *partitionedFanoutWriter) processRecord(ctx context.Context, writerCtx c
}

partitionPath := p.partitionPath(val.partitionRec)
rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, val.partitionValues, dataFilesChannel)
rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, val.partitionValues, dataFilesChannel, p.writerFactory.currentSpec)
if err != nil {
partitionRecord.Release()

Expand Down
2 changes: 1 addition & 1 deletion table/pos_delete_partitioned_fanout_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ func (p *positionDeletePartitionedFanoutWriter) processBatch(ctx context.Context
if err != nil {
return err
}
rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, partitionContext.partitionData, dataFilesChannel)
rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, partitionContext.partitionData, dataFilesChannel, *p.metadata.PartitionSpecByID(int(partitionContext.specID)))
if err != nil {
return err
}
Expand Down
113 changes: 113 additions & 0 deletions table/pos_delete_spec_evolution_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package table_test

import (
"context"
"fmt"
"path/filepath"
"testing"

"github.com/apache/arrow-go/v18/arrow/array"
"github.com/apache/arrow-go/v18/arrow/memory"
"github.com/apache/iceberg-go"
iceio "github.com/apache/iceberg-go/io"
"github.com/apache/iceberg-go/table"
"github.com/stretchr/testify/require"
)

func TestPositionDeleteAfterSpecEvolution(t *testing.T) {
for _, tc := range []struct {
name string
mixed, unpartitioned, sharedPath bool
}{
{name: "old-spec"},
{name: "mixed-specs", mixed: true},
{name: "evolve-to-unpartitioned", mixed: true, unpartitioned: true},
// truncate[2](cat) -> truncate[4](cat), both named cat: "ab" gives the same partition path under both specs.
{name: "shared-partition-path", mixed: true, sharedPath: true},
} {
Comment on lines +38 to +44

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

in every case the two specs write different partition paths, so only the unit test covers the spec id in the writer key

better: add a case where they share one, eg: "truncate2" replaced by "truncate4", both named c, with cat = "ab" rows before and after

t.Run(tc.name, func(t *testing.T) {
ctx := context.Background()
location := filepath.ToSlash(t.TempDir())
schema := iceberg.NewSchema(0,
iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true},
iceberg.NestedField{ID: 2, Name: "cat", Type: iceberg.PrimitiveTypes.String},
iceberg.NestedField{ID: 3, Name: "region", Type: iceberg.PrimitiveTypes.String})
var transform iceberg.Transform = iceberg.IdentityTransform{}
if tc.sharedPath {
transform = iceberg.TruncateTransform{Width: 2}
}
spec := iceberg.NewPartitionSpec(iceberg.PartitionField{SourceIDs: []int{2}, FieldID: 1000, Transform: transform, Name: "cat"})
meta, err := table.NewMetadata(schema, &spec, table.UnsortedSortOrder, location, iceberg.Properties{table.PropertyFormatVersion: "2", table.WriteDeleteModeKey: table.WriteModeMergeOnRead})
require.NoError(t, err)
fs := func(context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil }
metaLoc := location + "/metadata/v1.metadata.json"
cat := &concurrentTestCatalog{metadata: meta, location: metaLoc, fsF: fs}
tbl := table.New(table.Identifier{"db", "spec_delete"}, meta, metaLoc, fs, cat)
arrowSchema, err := table.SchemaToArrowSchema(schema, nil, false, false)
require.NoError(t, err)
appendJSON := func(rows string) {
data, e := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{rows})
require.NoError(t, e)
defer data.Release()
tbl, e = tbl.Append(ctx, array.NewTableReader(data, -1), nil)
require.NoError(t, e)
}
catVal := "a"
if tc.sharedPath {
catVal = "ab"
}
rows := fmt.Sprintf(`[{"id":1,"cat":%[1]q,"region":"x"},{"id":2,"cat":%[1]q,"region":"x"}]`, catVal)
moreRows := fmt.Sprintf(`[{"id":1,"cat":%[1]q,"region":"x"},{"id":3,"cat":%[1]q,"region":"x"}]`, catVal)
appendJSON(rows)
evolve := tbl.NewTransaction()
if tc.sharedPath {
require.NoError(t, evolve.UpdateSpec(true).RemoveField("cat").AddField("cat", iceberg.TruncateTransform{Width: 4}, "cat").Commit())
} else if tc.unpartitioned {
require.NoError(t, evolve.UpdateSpec(true).RemoveField("cat").Commit())
} else {
require.NoError(t, evolve.UpdateSpec(true).AddIdentity("region").Commit())
}
tbl, err = evolve.Commit(ctx)
require.NoError(t, err)
if tc.mixed {
appendJSON(moreRows)
}
tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(1)), nil)
require.NoError(t, err)
want := []int64{2}
if tc.mixed {
want = []int64{2, 3}
}
require.Equal(t, want, idsInTable(t, tbl))
tasks, err := tbl.Scan().PlanFiles(ctx)
require.NoError(t, err)
expectedTasks := 1
if tc.mixed {
expectedTasks = 2
}
require.Len(t, tasks, expectedTasks)
for _, task := range tasks {
require.Len(t, task.DeleteFiles, 1)
require.Equal(t, task.File.SpecID(), task.DeleteFiles[0].SpecID())
require.Equal(t, task.File.Partition(), task.DeleteFiles[0].Partition())
}
})
}
}
34 changes: 22 additions & 12 deletions table/rolling_data_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ func newWriterFactory(rootLocation string, args recordWritingArgs, meta *Metadat
}

func (w *writerFactory) openFileWriter(ctx context.Context, partitionPath string,
partitionValues map[int]any, partitionID int, fileCount int, arrowSchema *arrow.Schema,
partitionValues map[int]any, partitionID int, fileCount int, arrowSchema *arrow.Schema, spec iceberg.PartitionSpec,
) (tblutils.FileWriter, error) {
if arrowSchema == nil {
arrowSchema = w.arrowSchema
Expand All @@ -311,23 +311,30 @@ func (w *writerFactory) openFileWriter(ctx context.Context, partitionPath string
VariantFieldIDs: w.variantFieldIDs,
WriteProps: w.writeProps,
RowGroupBytes: w.rowGroupBytes,
Spec: w.currentSpec,
Spec: spec,
Content: w.content,
EqualityFieldIDs: w.equalityFieldIDs,
SortOrderID: w.sortOrderID,
}, arrowSchema)
}

type rollingWriterKey struct {
specID int
path string
}

// RollingDataWriter writes Arrow records for a specific partition, rolling to
// new data files when the actual compressed file size reaches the target.
type RollingDataWriter struct {
key rollingWriterKey
partitionKey string
partitionID int
fileCount atomic.Int64
recordCh chan arrow.RecordBatch
errorCh chan error
factory *writerFactory
partitionValues map[int]any
spec iceberg.PartitionSpec
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
Expand All @@ -351,16 +358,18 @@ func (w *writerFactory) recordQueueCapacity() int {
return rollingDataWriterQueueCapacity
}

func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile) *RollingDataWriter {
func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile, spec iceberg.PartitionSpec) *RollingDataWriter {
ctx, cancel := context.WithCancel(ctx)
partitionID := int(w.partitionIDCounter.Add(1) - 1)
writer := &RollingDataWriter{
key: rollingWriterKey{spec.ID(), partition},
partitionKey: partition,
partitionID: partitionID,
recordCh: make(chan arrow.RecordBatch, w.recordQueueCapacity()),
errorCh: make(chan error, 1),
factory: w,
partitionValues: partitionValues,
spec: spec,
ctx: ctx,
cancel: cancel,
}
Expand All @@ -371,8 +380,9 @@ func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition stri
return writer
}

func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile) (*RollingDataWriter, error) {
if existing, ok := w.writers.Load(partition); ok {
func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile, spec iceberg.PartitionSpec) (*RollingDataWriter, error) {
key := rollingWriterKey{spec.ID(), partition}
if existing, ok := w.writers.Load(key); ok {
if writer, ok := existing.(*RollingDataWriter); ok {
return writer, nil
}
Expand All @@ -385,16 +395,16 @@ func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partit

// Recheck after acquiring the lock in case another goroutine created the
// writer while this goroutine was waiting.
if existing, ok := w.writers.Load(partition); ok {
if existing, ok := w.writers.Load(key); ok {
if writer, ok := existing.(*RollingDataWriter); ok {
return writer, nil
}

return nil, fmt.Errorf("invalid writer type for partition: %s", partition)
}

writer := w.newRollingDataWriter(ctx, partition, partitionValues, outputDataFilesCh)
w.writers.Store(partition, writer)
writer := w.newRollingDataWriter(ctx, partition, partitionValues, outputDataFilesCh, spec)
w.writers.Store(key, writer)

return writer, nil
}
Expand Down Expand Up @@ -432,7 +442,7 @@ func (r *RollingDataWriter) Add(record arrow.RecordBatch) error {
func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) {
defer r.wg.Done()
defer close(r.errorCh)
defer r.factory.writers.CompareAndDelete(r.partitionKey, r)
defer r.factory.writers.CompareAndDelete(r.key, r)

alloc := compute.GetAllocator(r.ctx)

Expand Down Expand Up @@ -529,7 +539,7 @@ func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) {
fileCount := int(r.fileCount.Add(1))
var err error
currentWriter, err = r.factory.openFileWriter(
r.ctx, r.partitionKey, r.partitionValues, r.partitionID, fileCount, fileArrowSchema)
r.ctx, r.partitionKey, r.partitionValues, r.partitionID, fileCount, fileArrowSchema, r.spec)

return err
}
Expand Down Expand Up @@ -786,7 +796,7 @@ func (r *RollingDataWriter) abort() {

func (r *RollingDataWriter) closeAndWait() error {
r.closeInput()
r.factory.writers.Delete(r.partitionKey)
r.factory.writers.CompareAndDelete(r.key, r)
r.wg.Wait()

if err := <-r.errorCh; err != nil {
Expand All @@ -802,7 +812,7 @@ func (r *RollingDataWriter) closeAndWait() error {

func (r *RollingDataWriter) abortAndWait() {
r.abort()
r.factory.writers.Delete(r.partitionKey)
r.factory.writers.CompareAndDelete(r.key, r)
r.wg.Wait()
}

Expand Down
Loading
Loading