diff --git a/table/arrow_utils.go b/table/arrow_utils.go index fa6152ae3..ecf8cd3ff 100644 --- a/table/arrow_utils.go +++ b/table/arrow_utils.go @@ -2345,7 +2345,7 @@ func unpartitionedWrite(ctx context.Context, factory *writerFactory, records ite defer close(outputCh) defer factory.stopCount() - writer := factory.newRollingDataWriter(writerCtx, "", nil, outputCh) + writer := factory.newRollingDataWriter(writerCtx, "", nil, outputCh, factory.currentSpec) for rec, err := range records { if err != nil { errCh <- err @@ -2444,7 +2444,18 @@ func positionDeleteRecordsToDataFiles(ctx context.Context, rootLocation string, targetFileSize := int64(meta.props.GetInt(WriteDeleteTargetFileSizeBytesKey, WriteDeleteTargetFileSizeBytesDefault)) - if latestMetadata.PartitionSpec().IsUnpartitioned() { + currentSpec := latestMetadata.PartitionSpec() + // The unpartitioned writer stamps the current spec on every file, so it is + // safe only when every target file was written under that spec. + useUnpartitionedWriter := currentSpec.IsUnpartitioned() + for _, partition := range partitionContextByFilePath { + if int(partition.specID) != currentSpec.ID() { + useUnpartitionedWriter = false + + break + } + } + if useUnpartitionedWriter { cw := newConcurrentDataFileWriter(func(rootLocation string, fs iceio.WriteFileIO, meta *MetadataBuilder, props iceberg.Properties, opts ...dataFileWriterOption) (dataFileWriter, error) { return newPositionDeleteWriter(rootLocation, fs, meta, props, opts...) }, withSchemaSanitization(false)) diff --git a/table/clustered_writer.go b/table/clustered_writer.go index 1e52dfcf4..cfcd13559 100644 --- a/table/clustered_writer.go +++ b/table/clustered_writer.go @@ -127,7 +127,7 @@ func clusteredPartitionedWrite( } partitionPath := extractionPlan.pathPlan.format(part.partitionRec) currentWriter = factory.newRollingDataWriter( - ctx, partitionPath, part.partitionValues, outputCh) + ctx, partitionPath, part.partitionValues, outputCh, factory.currentSpec) currentRec = part.partitionRec } diff --git a/table/delete_parquet_settings_test.go b/table/delete_parquet_settings_test.go index 30ef9087b..e138297b6 100644 --- a/table/delete_parquet_settings_test.go +++ b/table/delete_parquet_settings_test.go @@ -111,7 +111,7 @@ func TestDeleteParquetSettingsOnDisk(t *testing.T) { defer factory.stopCount() writerProps = factory.writeProps.([]parquet.WriterProperty) rowGroupBytes = factory.rowGroupBytes - writer, err := factory.openFileWriter(ctx, "", nil, 0, 1, nil) + writer, err := factory.openFileWriter(ctx, "", nil, 0, 1, nil, factory.currentSpec) require.NoError(t, err) for _, rec := range batches { require.NoError(t, writer.Write(rec)) diff --git a/table/parquet_bloom_properties_test.go b/table/parquet_bloom_properties_test.go index fea1c831f..2409dc384 100644 --- a/table/parquet_bloom_properties_test.go +++ b/table/parquet_bloom_properties_test.go @@ -81,7 +81,7 @@ func TestParquetBloomLogicalPathsOnDisk(t *testing.T) { }, builder, fileSchema, 1048576, withFactoryFileSchema(fileSchema)) require.NoError(t, err) defer factory.stopCount() - writer, err := factory.openFileWriter(t.Context(), "", nil, 0, 1, nil) + writer, err := factory.openFileWriter(t.Context(), "", nil, 0, 1, nil, factory.currentSpec) require.NoError(t, err) require.NoError(t, writer.Write(batch)) df, err = writer.Close() diff --git a/table/parquet_column_stats_test.go b/table/parquet_column_stats_test.go index a0c221667..bc5086b67 100644 --- a/table/parquet_column_stats_test.go +++ b/table/parquet_column_stats_test.go @@ -110,7 +110,7 @@ func writeStatsSettingsFile(t *testing.T, mode string, props iceberg.Properties) require.NoError(t, err) defer factory.closeAll() output := make(chan iceberg.DataFile, 1) - writer := factory.newRollingDataWriter(t.Context(), "", nil, output) + writer := factory.newRollingDataWriter(t.Context(), "", nil, output, factory.currentSpec) require.NoError(t, writer.Add(record)) record.Release() require.NoError(t, writer.closeAndWait()) @@ -267,7 +267,7 @@ func writeStatsSettingsDeleteFile(t *testing.T, mode string, content iceberg.Man require.NoError(t, err) defer factory.closeAll() output := make(chan iceberg.DataFile, 1) - writer := factory.newRollingDataWriter(t.Context(), "", nil, output) + writer := factory.newRollingDataWriter(t.Context(), "", nil, output, factory.currentSpec) require.NoError(t, writer.Add(record)) record.Release() require.NoError(t, writer.closeAndWait()) diff --git a/table/partitioned_fanout_writer.go b/table/partitioned_fanout_writer.go index 74af816bf..d30b2c932 100644 --- a/table/partitioned_fanout_writer.go +++ b/table/partitioned_fanout_writer.go @@ -214,7 +214,7 @@ func (p *partitionedFanoutWriter) processRecord(ctx context.Context, writerCtx c } partitionPath := p.partitionPath(val.partitionRec) - rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, val.partitionValues, dataFilesChannel) + rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, val.partitionValues, dataFilesChannel, p.writerFactory.currentSpec) if err != nil { partitionRecord.Release() diff --git a/table/pos_delete_partitioned_fanout_writer.go b/table/pos_delete_partitioned_fanout_writer.go index 3f29db5c5..05215b82b 100644 --- a/table/pos_delete_partitioned_fanout_writer.go +++ b/table/pos_delete_partitioned_fanout_writer.go @@ -117,7 +117,7 @@ func (p *positionDeletePartitionedFanoutWriter) processBatch(ctx context.Context if err != nil { return err } - rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, partitionContext.partitionData, dataFilesChannel) + rollingDataWriter, err := p.writerFactory.getOrCreateRollingDataWriter(writerCtx, partitionPath, partitionContext.partitionData, dataFilesChannel, *p.metadata.PartitionSpecByID(int(partitionContext.specID))) if err != nil { return err } diff --git a/table/pos_delete_spec_evolution_test.go b/table/pos_delete_spec_evolution_test.go new file mode 100644 index 000000000..e3493a642 --- /dev/null +++ b/table/pos_delete_spec_evolution_test.go @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package table_test + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/apache/arrow-go/v18/arrow/array" + "github.com/apache/arrow-go/v18/arrow/memory" + "github.com/apache/iceberg-go" + iceio "github.com/apache/iceberg-go/io" + "github.com/apache/iceberg-go/table" + "github.com/stretchr/testify/require" +) + +func TestPositionDeleteAfterSpecEvolution(t *testing.T) { + for _, tc := range []struct { + name string + mixed, unpartitioned, sharedPath bool + }{ + {name: "old-spec"}, + {name: "mixed-specs", mixed: true}, + {name: "evolve-to-unpartitioned", mixed: true, unpartitioned: true}, + // truncate[2](cat) -> truncate[4](cat), both named cat: "ab" gives the same partition path under both specs. + {name: "shared-partition-path", mixed: true, sharedPath: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + location := filepath.ToSlash(t.TempDir()) + schema := iceberg.NewSchema(0, + iceberg.NestedField{ID: 1, Name: "id", Type: iceberg.PrimitiveTypes.Int64, Required: true}, + iceberg.NestedField{ID: 2, Name: "cat", Type: iceberg.PrimitiveTypes.String}, + iceberg.NestedField{ID: 3, Name: "region", Type: iceberg.PrimitiveTypes.String}) + var transform iceberg.Transform = iceberg.IdentityTransform{} + if tc.sharedPath { + transform = iceberg.TruncateTransform{Width: 2} + } + spec := iceberg.NewPartitionSpec(iceberg.PartitionField{SourceIDs: []int{2}, FieldID: 1000, Transform: transform, Name: "cat"}) + meta, err := table.NewMetadata(schema, &spec, table.UnsortedSortOrder, location, iceberg.Properties{table.PropertyFormatVersion: "2", table.WriteDeleteModeKey: table.WriteModeMergeOnRead}) + require.NoError(t, err) + fs := func(context.Context) (iceio.IO, error) { return iceio.LocalFS{}, nil } + metaLoc := location + "/metadata/v1.metadata.json" + cat := &concurrentTestCatalog{metadata: meta, location: metaLoc, fsF: fs} + tbl := table.New(table.Identifier{"db", "spec_delete"}, meta, metaLoc, fs, cat) + arrowSchema, err := table.SchemaToArrowSchema(schema, nil, false, false) + require.NoError(t, err) + appendJSON := func(rows string) { + data, e := array.TableFromJSON(memory.DefaultAllocator, arrowSchema, []string{rows}) + require.NoError(t, e) + defer data.Release() + tbl, e = tbl.Append(ctx, array.NewTableReader(data, -1), nil) + require.NoError(t, e) + } + catVal := "a" + if tc.sharedPath { + catVal = "ab" + } + rows := fmt.Sprintf(`[{"id":1,"cat":%[1]q,"region":"x"},{"id":2,"cat":%[1]q,"region":"x"}]`, catVal) + moreRows := fmt.Sprintf(`[{"id":1,"cat":%[1]q,"region":"x"},{"id":3,"cat":%[1]q,"region":"x"}]`, catVal) + appendJSON(rows) + evolve := tbl.NewTransaction() + if tc.sharedPath { + require.NoError(t, evolve.UpdateSpec(true).RemoveField("cat").AddField("cat", iceberg.TruncateTransform{Width: 4}, "cat").Commit()) + } else if tc.unpartitioned { + require.NoError(t, evolve.UpdateSpec(true).RemoveField("cat").Commit()) + } else { + require.NoError(t, evolve.UpdateSpec(true).AddIdentity("region").Commit()) + } + tbl, err = evolve.Commit(ctx) + require.NoError(t, err) + if tc.mixed { + appendJSON(moreRows) + } + tbl, err = tbl.Delete(ctx, iceberg.EqualTo(iceberg.Reference("id"), int64(1)), nil) + require.NoError(t, err) + want := []int64{2} + if tc.mixed { + want = []int64{2, 3} + } + require.Equal(t, want, idsInTable(t, tbl)) + tasks, err := tbl.Scan().PlanFiles(ctx) + require.NoError(t, err) + expectedTasks := 1 + if tc.mixed { + expectedTasks = 2 + } + require.Len(t, tasks, expectedTasks) + for _, task := range tasks { + require.Len(t, task.DeleteFiles, 1) + require.Equal(t, task.File.SpecID(), task.DeleteFiles[0].SpecID()) + require.Equal(t, task.File.Partition(), task.DeleteFiles[0].Partition()) + } + }) + } +} diff --git a/table/rolling_data_writer.go b/table/rolling_data_writer.go index e133fb25f..2805d4905 100644 --- a/table/rolling_data_writer.go +++ b/table/rolling_data_writer.go @@ -284,7 +284,7 @@ func newWriterFactory(rootLocation string, args recordWritingArgs, meta *Metadat } func (w *writerFactory) openFileWriter(ctx context.Context, partitionPath string, - partitionValues map[int]any, partitionID int, fileCount int, arrowSchema *arrow.Schema, + partitionValues map[int]any, partitionID int, fileCount int, arrowSchema *arrow.Schema, spec iceberg.PartitionSpec, ) (tblutils.FileWriter, error) { if arrowSchema == nil { arrowSchema = w.arrowSchema @@ -311,16 +311,22 @@ func (w *writerFactory) openFileWriter(ctx context.Context, partitionPath string VariantFieldIDs: w.variantFieldIDs, WriteProps: w.writeProps, RowGroupBytes: w.rowGroupBytes, - Spec: w.currentSpec, + Spec: spec, Content: w.content, EqualityFieldIDs: w.equalityFieldIDs, SortOrderID: w.sortOrderID, }, arrowSchema) } +type rollingWriterKey struct { + specID int + path string +} + // RollingDataWriter writes Arrow records for a specific partition, rolling to // new data files when the actual compressed file size reaches the target. type RollingDataWriter struct { + key rollingWriterKey partitionKey string partitionID int fileCount atomic.Int64 @@ -328,6 +334,7 @@ type RollingDataWriter struct { errorCh chan error factory *writerFactory partitionValues map[int]any + spec iceberg.PartitionSpec ctx context.Context cancel context.CancelFunc wg sync.WaitGroup @@ -351,16 +358,18 @@ func (w *writerFactory) recordQueueCapacity() int { return rollingDataWriterQueueCapacity } -func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile) *RollingDataWriter { +func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile, spec iceberg.PartitionSpec) *RollingDataWriter { ctx, cancel := context.WithCancel(ctx) partitionID := int(w.partitionIDCounter.Add(1) - 1) writer := &RollingDataWriter{ + key: rollingWriterKey{spec.ID(), partition}, partitionKey: partition, partitionID: partitionID, recordCh: make(chan arrow.RecordBatch, w.recordQueueCapacity()), errorCh: make(chan error, 1), factory: w, partitionValues: partitionValues, + spec: spec, ctx: ctx, cancel: cancel, } @@ -371,8 +380,9 @@ func (w *writerFactory) newRollingDataWriter(ctx context.Context, partition stri return writer } -func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile) (*RollingDataWriter, error) { - if existing, ok := w.writers.Load(partition); ok { +func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partition string, partitionValues map[int]any, outputDataFilesCh chan<- iceberg.DataFile, spec iceberg.PartitionSpec) (*RollingDataWriter, error) { + key := rollingWriterKey{spec.ID(), partition} + if existing, ok := w.writers.Load(key); ok { if writer, ok := existing.(*RollingDataWriter); ok { return writer, nil } @@ -385,7 +395,7 @@ func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partit // Recheck after acquiring the lock in case another goroutine created the // writer while this goroutine was waiting. - if existing, ok := w.writers.Load(partition); ok { + if existing, ok := w.writers.Load(key); ok { if writer, ok := existing.(*RollingDataWriter); ok { return writer, nil } @@ -393,8 +403,8 @@ func (w *writerFactory) getOrCreateRollingDataWriter(ctx context.Context, partit return nil, fmt.Errorf("invalid writer type for partition: %s", partition) } - writer := w.newRollingDataWriter(ctx, partition, partitionValues, outputDataFilesCh) - w.writers.Store(partition, writer) + writer := w.newRollingDataWriter(ctx, partition, partitionValues, outputDataFilesCh, spec) + w.writers.Store(key, writer) return writer, nil } @@ -432,7 +442,7 @@ func (r *RollingDataWriter) Add(record arrow.RecordBatch) error { func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) { defer r.wg.Done() defer close(r.errorCh) - defer r.factory.writers.CompareAndDelete(r.partitionKey, r) + defer r.factory.writers.CompareAndDelete(r.key, r) alloc := compute.GetAllocator(r.ctx) @@ -529,7 +539,7 @@ func (r *RollingDataWriter) stream(outputDataFilesCh chan<- iceberg.DataFile) { fileCount := int(r.fileCount.Add(1)) var err error currentWriter, err = r.factory.openFileWriter( - r.ctx, r.partitionKey, r.partitionValues, r.partitionID, fileCount, fileArrowSchema) + r.ctx, r.partitionKey, r.partitionValues, r.partitionID, fileCount, fileArrowSchema, r.spec) return err } @@ -786,7 +796,7 @@ func (r *RollingDataWriter) abort() { func (r *RollingDataWriter) closeAndWait() error { r.closeInput() - r.factory.writers.Delete(r.partitionKey) + r.factory.writers.CompareAndDelete(r.key, r) r.wg.Wait() if err := <-r.errorCh; err != nil { @@ -802,7 +812,7 @@ func (r *RollingDataWriter) closeAndWait() error { func (r *RollingDataWriter) abortAndWait() { r.abort() - r.factory.writers.Delete(r.partitionKey) + r.factory.writers.CompareAndDelete(r.key, r) r.wg.Wait() } diff --git a/table/rolling_data_writer_test.go b/table/rolling_data_writer_test.go index 15f5e0aec..14d274c54 100644 --- a/table/rolling_data_writer_test.go +++ b/table/rolling_data_writer_test.go @@ -80,7 +80,7 @@ func BenchmarkWriterFactoryCacheHit(b *testing.B) { partitions := make([]string, partitionCount) for i := range partitionCount { partitions[i] = strconv.Itoa(i) - factory.writers.Store(partitions[i], &RollingDataWriter{}) + factory.writers.Store(rollingWriterKey{factory.currentSpec.ID(), partitions[i]}, &RollingDataWriter{}) } b.ReportAllocs() @@ -88,7 +88,7 @@ func BenchmarkWriterFactoryCacheHit(b *testing.B) { var i int for pb.Next() { writer, err := factory.getOrCreateRollingDataWriter( - context.Background(), partitions[i%partitionCount], nil, nil) + context.Background(), partitions[i%partitionCount], nil, nil, factory.currentSpec) if err != nil || writer == nil { b.Fatalf("get existing writer: writer=%v, err=%v", writer, err) } @@ -178,7 +178,7 @@ func (s *RollingDataWriterTestSuite) TestSingleFileUnderTarget() { defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 10) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) record := s.buildRecord(arrSchema, 5) defer record.Release() @@ -207,7 +207,7 @@ func (s *RollingDataWriterTestSuite) TestRollsMultipleFiles() { defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 100) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) totalRows := int64(0) for range 10 { @@ -280,7 +280,7 @@ func (s *RollingDataWriterTestSuite) TestCloseAllFinishesQueuedRecordsWithoutCan writerCtx, cancel := context.WithCancel(s.ctx) defer cancel() - writer, err := factory.getOrCreateRollingDataWriter(writerCtx, "", nil, outputCh) + writer, err := factory.getOrCreateRollingDataWriter(writerCtx, "", nil, outputCh, factory.currentSpec) s.Require().NoError(err) const totalBatches = 4 const rowsPerBatch = 25 @@ -319,7 +319,7 @@ func (s *RollingDataWriterTestSuite) TestConcurrentGetOrCreateCreatesOneWriter() for range goroutineCount { wg.Go(func() { <-start - writer, err := factory.getOrCreateRollingDataWriter(s.ctx, "partition", nil, outputCh) + writer, err := factory.getOrCreateRollingDataWriter(s.ctx, "partition", nil, outputCh, factory.currentSpec) results <- struct { writer *RollingDataWriter err error @@ -345,6 +345,26 @@ func (s *RollingDataWriterTestSuite) TestConcurrentGetOrCreateCreatesOneWriter() s.Require().NoError(expected.closeAndWait()) } +func (s *RollingDataWriterTestSuite) TestWriterKeysIncludeSpecAndAreRemoved() { + factory := &writerFactory{} + output := make(chan iceberg.DataFile) + spec0 := iceberg.NewPartitionSpecID(0) + spec1 := iceberg.NewPartitionSpecID(1) + first, err := factory.getOrCreateRollingDataWriter(s.ctx, "same-path", nil, output, spec0) + s.Require().NoError(err) + second, err := factory.getOrCreateRollingDataWriter(s.ctx, "same-path", nil, output, spec1) + s.Require().NoError(err) + s.NotSame(first, second) + s.Require().NoError(first.closeAndWait()) + _, firstExists := factory.writers.Load(first.key) + _, secondExists := factory.writers.Load(second.key) + s.False(firstExists) + s.True(secondExists) + second.abortAndWait() + _, secondExists = factory.writers.Load(second.key) + s.False(secondExists) +} + func (s *RollingDataWriterTestSuite) TestAbortAndWaitCancelsContext() { arrSchema := arrow.NewSchema([]arrow.Field{ {Name: "id", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, @@ -359,7 +379,7 @@ func (s *RollingDataWriterTestSuite) TestAbortAndWaitCancelsContext() { writerCtx, cancel := context.WithCancel(s.ctx) defer cancel() - writer, err := factory.getOrCreateRollingDataWriter(writerCtx, "", nil, outputCh) + writer, err := factory.getOrCreateRollingDataWriter(writerCtx, "", nil, outputCh, factory.currentSpec) s.Require().NoError(err) record := s.buildRecord(arrSchema, 5) s.Require().NoError(writer.Add(record)) @@ -479,7 +499,7 @@ func (s *RollingDataWriterTestSuite) TestWriterFactoryDoesNotStampSortOrderID() defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 10) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) record := s.buildRecord(arrSchema, 5) defer record.Release() @@ -526,7 +546,7 @@ func (s *RollingDataWriterTestSuite) TestStreamErrorPathUsesAbort() { defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 10) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) // Send a record with an incompatible schema to trigger a // ToRequestedSchema error, exercising the deferred Abort() path. @@ -590,7 +610,7 @@ func (s *RollingDataWriterTestSuite) TestStreamErrorRemovesWriterFromFactory() { defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 10) - writer, err := factory.getOrCreateRollingDataWriter(s.ctx, "", nil, outputCh) + writer, err := factory.getOrCreateRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) s.Require().NoError(err) badSchema := arrow.NewSchema([]arrow.Field{ @@ -605,7 +625,7 @@ func (s *RollingDataWriterTestSuite) TestStreamErrorRemovesWriterFromFactory() { s.Require().NoError(writer.Add(badRecord)) writer.wg.Wait() - _, ok := factory.writers.Load("") + _, ok := factory.writers.Load(writer.key) s.False(ok, "failed writer should be removed from partition writer cache") err, ok = <-writer.errorCh @@ -724,7 +744,7 @@ func (s *RollingDataWriterTestSuite) TestPartitionedDataLocations() { defer factory.closeAll() outputCh := make(chan iceberg.DataFile, 1) - writer := factory.newRollingDataWriter(s.ctx, partition, nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, partition, nil, outputCh, factory.currentSpec) record := s.buildRecord(arrSchema, 5) defer record.Release() @@ -823,7 +843,7 @@ func (s *RollingDataWriterTestSuite) TestSortsRowsBeforeWriting() { defer record.Release() outputCh := make(chan iceberg.DataFile, 10) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) s.Require().NoError(writer.Add(record)) s.Require().NoError(writer.closeAndWait()) close(outputCh) @@ -907,7 +927,7 @@ func (s *RollingDataWriterTestSuite) TestUnsortedOrderIsNoOp() { defer record.Release() outputCh := make(chan iceberg.DataFile, 10) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) s.Require().NoError(writer.Add(record)) s.Require().NoError(writer.closeAndWait()) close(outputCh) @@ -997,7 +1017,7 @@ func (s *RollingDataWriterTestSuite) TestStreamRecoversWriterClosePanic() { factory.format = panicOnCloseFormat{FileFormat: factory.format, bytesWritten: tt.bytesWritten} outputCh := make(chan iceberg.DataFile, 1) - writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh) + writer := factory.newRollingDataWriter(s.ctx, "", nil, outputCh, factory.currentSpec) record := s.buildRecord(arrSchema, 5) defer record.Release() diff --git a/table/write_read_tuning_test.go b/table/write_read_tuning_test.go index 4ab715e1f..115da6a1a 100644 --- a/table/write_read_tuning_test.go +++ b/table/write_read_tuning_test.go @@ -162,12 +162,12 @@ func TestRecordBatchBufferSizeReachesRollingWriter(t *testing.T) { outputCh := make(chan iceberg.DataFile, 1) overridden := newTuningTestWriterFactory(t, 3) - w := overridden.newRollingDataWriter(t.Context(), "", nil, outputCh) + w := overridden.newRollingDataWriter(t.Context(), "", nil, outputCh, overridden.currentSpec) assert.Equal(t, 3, cap(w.recordCh)) w.abortAndWait() defaulted := newTuningTestWriterFactory(t, 0) - dw := defaulted.newRollingDataWriter(t.Context(), "", nil, outputCh) + dw := defaulted.newRollingDataWriter(t.Context(), "", nil, outputCh, defaulted.currentSpec) assert.Equal(t, rollingDataWriterQueueCapacity, cap(dw.recordCh)) dw.abortAndWait() } diff --git a/table/writer_schema_projection_test.go b/table/writer_schema_projection_test.go index 0ded9753b..a3d3f6c31 100644 --- a/table/writer_schema_projection_test.go +++ b/table/writer_schema_projection_test.go @@ -476,7 +476,7 @@ func TestRollingDataWriterReusesExactBatch(t *testing.T) { factory.format = format output := make(chan iceberg.DataFile, 1) - writer := factory.newRollingDataWriter(t.Context(), "", nil, output) + writer := factory.newRollingDataWriter(t.Context(), "", nil, output, factory.currentSpec) record := projectionTestRecord(t, mem, arrowSchema, []byte(`{"id": 1}`)) defer record.Release() diff --git a/website/src/releases.md b/website/src/releases.md index 12cc5f456..e5be8c140 100644 --- a/website/src/releases.md +++ b/website/src/releases.md @@ -148,6 +148,7 @@ Per-release notes (highlights, breaking changes, contributors) are published on - **Commit rejections now match `table.ErrCommitRejected`.** SQL, Glue, Hive and Hadoop failures while preparing metadata, plus REST commit responses 400, 401, 403 and 404, leave transactions committable (after a REST 404, the next `Commit` fails when it reloads the table). Existing `errors.Is` matches keep working, but SQL, Glue, Hive and Hadoop rejection messages now start with `commit rejected: `, so code that matches error text may need updating. - **Float comparisons against literals beyond the float32 range follow IEEE 754.** On a float32 column, `x < 1e100` no longer returns NaN, `+Inf` or NULL rows, and `NOT(x > 1e100)` no longer returns NULL rows. An infinite literal now binds to a float32 infinity, so `x = +Inf` matches `+Inf` rows. Java still folds these comparisons to always true or false, see [apache/iceberg#18700](https://github.com/apache/iceberg/issues/18700). - **Orphan cleanup refreshes metadata before deletion.** Non-dry-run `DeleteOrphanFiles` requires a catalog-backed table and refreshes it before planning and again before deleting files. `ExecuteOrphanCleanup` also refreshes before deleting from a non-empty plan. If a saved plan is stale, refresh or reload the table before planning again. Dry runs still work without a catalog. +- **Position deletes use the data file's partition spec.** After partition spec evolution, `Delete` on a merge-on-read table wrote v2 position delete files with the current spec instead of the target data file's spec. Scans and re-running `Delete` on such a table can fail with `invalid metadata: position delete file ... uses partition spec N, but data file ... uses partition spec M`; some deletes are instead silently missed, so the rows stay visible, and re-running those deletes applies them. New deletes carry the data file's spec. For the mismatch error, remove the delete files whose spec ID differs from their data file's with `Transaction.NewRewrite` (`RewriteFiles.DeleteFile`), then re-run those deletes; until then the rows they deleted are visible again. ## Verifying and producing releases