From ca1193badf477e646fa06fdf6037ca72a5c0704c Mon Sep 17 00:00:00 2001 From: Alex Stephen Date: Thu, 5 Nov 2026 19:57:13 +0000 Subject: [PATCH] Core: Resolve last-added -1 to a re-added schema, spec, or sort order This was something I discovered working on the Terraform provider. Adding a schema, partition spec, or sort order that already exists in the table and then setting it current or default with -1 failed with "no schema has been added". Clients that send the full desired state, such as declarative tools that revert a column or a partition spec, hit this on every revert. The re-added entry is now the last added one. The change log records -1 only when the latest add of that type in the changes has the same ID, so a server replaying the changes selects the same entry. --- .../org/apache/iceberg/TableMetadata.java | 47 +++++---- .../org/apache/iceberg/TestTableMetadata.java | 98 +++++++++++++++++++ 2 files changed, 120 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 89427e4c6943..a3d30a042c47 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.ToIntFunction; import java.util.stream.Collectors; import java.util.stream.Stream; import org.apache.iceberg.encryption.EncryptedKey; @@ -1116,7 +1117,8 @@ public Builder setCurrentSchema(int schemaId) { this.currentSchemaId = schemaId; - if (lastAddedSchemaId != null && lastAddedSchemaId == schemaId) { + if (isLastAddedChange( + MetadataUpdate.AddSchema.class, added -> added.schema().schemaId(), schemaId)) { changes.add(new MetadataUpdate.SetCurrentSchema(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetCurrentSchema(schemaId)); @@ -1148,7 +1150,8 @@ public Builder setDefaultPartitionSpec(int specId) { } this.defaultSpecId = specId; - if (lastAddedSpecId != null && lastAddedSpecId == specId) { + if (isLastAddedChange( + MetadataUpdate.AddPartitionSpec.class, added -> added.spec().specId(), specId)) { changes.add(new MetadataUpdate.SetDefaultPartitionSpec(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetDefaultPartitionSpec(specId)); @@ -1217,7 +1220,8 @@ public Builder setDefaultSortOrder(int sortOrderId) { } this.defaultSortOrderId = sortOrderId; - if (lastAddedOrderId != null && lastAddedOrderId == sortOrderId) { + if (isLastAddedChange( + MetadataUpdate.AddSortOrder.class, added -> added.sortOrder().orderId(), sortOrderId)) { changes.add(new MetadataUpdate.SetDefaultSortOrder(LAST_ADDED)); } else { changes.add(new MetadataUpdate.SetDefaultSortOrder(sortOrderId)); @@ -1627,14 +1631,7 @@ private int addSchemaInternal(Schema schema, int newLastColumnId) { int newSchemaId = reuseOrCreateNewSchemaId(schema); boolean schemaFound = schemasById.containsKey(newSchemaId); if (schemaFound && newLastColumnId == lastColumnId) { - // the new spec and last column id is already current and no change is needed - // update lastAddedSchemaId if the schema was added in this set of changes (since it is now - // the last) - boolean isNewSchema = - lastAddedSchemaId != null - && changes(MetadataUpdate.AddSchema.class) - .anyMatch(added -> added.schema().schemaId() == newSchemaId); - this.lastAddedSchemaId = isNewSchema ? newSchemaId : null; + this.lastAddedSchemaId = newSchemaId; return newSchemaId; } @@ -1675,13 +1672,7 @@ private int reuseOrCreateNewSchemaId(Schema newSchema) { private int addPartitionSpecInternal(PartitionSpec spec) { int newSpecId = reuseOrCreateNewSpecId(spec); if (specsById.containsKey(newSpecId)) { - // update lastAddedSpecId if the spec was added in this set of changes (since it is now the - // last) - boolean isNewSpec = - lastAddedSpecId != null - && changes(MetadataUpdate.AddPartitionSpec.class) - .anyMatch(added -> added.spec().specId() == lastAddedSpecId); - this.lastAddedSpecId = isNewSpec ? newSpecId : null; + this.lastAddedSpecId = newSpecId; return newSpecId; } @@ -1722,13 +1713,7 @@ private int reuseOrCreateNewSpecId(PartitionSpec newSpec) { private int addSortOrderInternal(SortOrder order) { int newOrderId = reuseOrCreateNewSortOrderId(order); if (sortOrdersById.containsKey(newOrderId)) { - // update lastAddedOrderId if the order was added in this set of changes (since it is now - // the last) - boolean isNewOrder = - lastAddedOrderId != null - && changes(MetadataUpdate.AddSortOrder.class) - .anyMatch(added -> added.sortOrder().orderId() == lastAddedOrderId); - this.lastAddedOrderId = isNewOrder ? newOrderId : null; + this.lastAddedOrderId = newOrderId; return newOrderId; } @@ -1920,6 +1905,18 @@ private boolean isAddedSnapshot(long snapshotId) { .anyMatch(add -> add.snapshot().snapshotId() == snapshotId); } + private boolean isLastAddedChange( + Class addClass, ToIntFunction addedId, int id) { + for (int i = changes.size() - 1; i >= 0; i -= 1) { + MetadataUpdate change = changes.get(i); + if (addClass.isInstance(change)) { + return addedId.applyAsInt(addClass.cast(change)) == id; + } + } + + return false; + } + private Stream changes(Class updateClass) { return changes.stream().filter(updateClass::isInstance).map(updateClass::cast); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..b716a863a820 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -1710,6 +1710,104 @@ schema, new Schema(1, schema2.columns()), new Schema(2, schema3.columns())), assertThat(threeSchemaTable.lastColumnId()).isEqualTo(6); } + @Test + void setCurrentSchemaToReAddedHistoricalSchema() { + Schema original = new Schema(Types.NestedField.required(1, "x", Types.LongType.get())); + TableMetadata base = + TableMetadata.newTableMetadata( + original, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + TableMetadata updated = + base.updateSchema( + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.optional(2, "y", Types.StringType.get()))); + + TableMetadata reverted = + TableMetadata.buildFrom(updated).addSchema(original).setCurrentSchema(-1).build(); + + assertThat(reverted.currentSchemaId()).isEqualTo(base.currentSchemaId()); + assertThat(reverted.changes()).hasSize(updated.changes().size() + 1); + assertThat(Iterables.getLast(reverted.changes())) + .isInstanceOfSatisfying( + MetadataUpdate.SetCurrentSchema.class, + update -> assertThat(update.schemaId()).isEqualTo(base.currentSchemaId())); + } + + @Test + void setDefaultSpecToReAddedHistoricalSpec() { + Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get())); + TableMetadata base = + TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + TableMetadata updated = + base.updatePartitionSpec(PartitionSpec.builderFor(base.schema()).bucket("x", 4).build()); + + TableMetadata reverted = + TableMetadata.buildFrom(updated) + .addPartitionSpec(PartitionSpec.unpartitioned()) + .setDefaultPartitionSpec(-1) + .build(); + + assertThat(reverted.defaultSpecId()).isEqualTo(base.defaultSpecId()); + assertThat(reverted.changes()).hasSize(updated.changes().size() + 1); + assertThat(Iterables.getLast(reverted.changes())) + .isInstanceOfSatisfying( + MetadataUpdate.SetDefaultPartitionSpec.class, + update -> assertThat(update.specId()).isEqualTo(base.defaultSpecId())); + } + + @Test + void setDefaultSortOrderToReAddedHistoricalSortOrder() { + Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get())); + TableMetadata base = + TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + TableMetadata updated = + base.replaceSortOrder(SortOrder.builderFor(base.schema()).asc("x").build()); + + TableMetadata reverted = + TableMetadata.buildFrom(updated) + .addSortOrder(SortOrder.unsorted()) + .setDefaultSortOrder(-1) + .build(); + + assertThat(reverted.defaultSortOrderId()).isEqualTo(base.defaultSortOrderId()); + assertThat(reverted.changes()).hasSize(updated.changes().size() + 1); + assertThat(Iterables.getLast(reverted.changes())) + .isInstanceOfSatisfying( + MetadataUpdate.SetDefaultSortOrder.class, + update -> assertThat(update.sortOrderId()).isEqualTo(base.defaultSortOrderId())); + } + + @Test + void replayedChangesSetCurrentSchemaToReAddedSchema() { + Schema original = new Schema(Types.NestedField.required(1, "x", Types.LongType.get())); + TableMetadata base = + TableMetadata.newTableMetadata( + original, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + Schema first = + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.optional(2, "y", Types.StringType.get())); + Schema second = + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.optional(3, "z", Types.StringType.get())); + + TableMetadata updated = + TableMetadata.buildFrom(base) + .addSchema(first) + .addSchema(second) + .addSchema(first) + .setCurrentSchema(-1) + .build(); + + TableMetadata.Builder replay = TableMetadata.buildFrom(base); + updated.changes().forEach(change -> change.applyTo(replay)); + + assertThat(replay.build().schema().asStruct()).isEqualTo(updated.schema().asStruct()); + } + @Test public void testCreateV2MetadataThroughTableProperty() { Schema schema = new Schema(Types.NestedField.required(10, "x", Types.StringType.get()));