Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 22 additions & 25 deletions core/src/main/java/org/apache/iceberg/TableMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.ToIntFunction;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.iceberg.encryption.EncryptedKey;
Expand Down Expand Up @@ -1116,7 +1117,8 @@ public Builder setCurrentSchema(int schemaId) {

this.currentSchemaId = schemaId;

if (lastAddedSchemaId != null && lastAddedSchemaId == schemaId) {
if (isLastAddedChange(
MetadataUpdate.AddSchema.class, added -> added.schema().schemaId(), schemaId)) {
changes.add(new MetadataUpdate.SetCurrentSchema(LAST_ADDED));
} else {
changes.add(new MetadataUpdate.SetCurrentSchema(schemaId));
Expand Down Expand Up @@ -1148,7 +1150,8 @@ public Builder setDefaultPartitionSpec(int specId) {
}

this.defaultSpecId = specId;
if (lastAddedSpecId != null && lastAddedSpecId == specId) {
if (isLastAddedChange(
MetadataUpdate.AddPartitionSpec.class, added -> added.spec().specId(), specId)) {
changes.add(new MetadataUpdate.SetDefaultPartitionSpec(LAST_ADDED));
} else {
changes.add(new MetadataUpdate.SetDefaultPartitionSpec(specId));
Expand Down Expand Up @@ -1217,7 +1220,8 @@ public Builder setDefaultSortOrder(int sortOrderId) {
}

this.defaultSortOrderId = sortOrderId;
if (lastAddedOrderId != null && lastAddedOrderId == sortOrderId) {
if (isLastAddedChange(
MetadataUpdate.AddSortOrder.class, added -> added.sortOrder().orderId(), sortOrderId)) {
changes.add(new MetadataUpdate.SetDefaultSortOrder(LAST_ADDED));
} else {
changes.add(new MetadataUpdate.SetDefaultSortOrder(sortOrderId));
Expand Down Expand Up @@ -1627,14 +1631,7 @@ private int addSchemaInternal(Schema schema, int newLastColumnId) {
int newSchemaId = reuseOrCreateNewSchemaId(schema);
boolean schemaFound = schemasById.containsKey(newSchemaId);
if (schemaFound && newLastColumnId == lastColumnId) {
// the new spec and last column id is already current and no change is needed
// update lastAddedSchemaId if the schema was added in this set of changes (since it is now
// the last)
boolean isNewSchema =
lastAddedSchemaId != null
&& changes(MetadataUpdate.AddSchema.class)
.anyMatch(added -> added.schema().schemaId() == newSchemaId);
this.lastAddedSchemaId = isNewSchema ? newSchemaId : null;
this.lastAddedSchemaId = newSchemaId;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The equivalent reuse path in ViewMetadata still leaves lastAddedSchemaId unset or pointing at a different schema. Should we have a follow-up for views?

return newSchemaId;
}

Expand Down Expand Up @@ -1675,13 +1672,7 @@ private int reuseOrCreateNewSchemaId(Schema newSchema) {
private int addPartitionSpecInternal(PartitionSpec spec) {
int newSpecId = reuseOrCreateNewSpecId(spec);
if (specsById.containsKey(newSpecId)) {
// update lastAddedSpecId if the spec was added in this set of changes (since it is now the
// last)
boolean isNewSpec =
lastAddedSpecId != null
&& changes(MetadataUpdate.AddPartitionSpec.class)
.anyMatch(added -> added.spec().specId() == lastAddedSpecId);
this.lastAddedSpecId = isNewSpec ? newSpecId : null;
this.lastAddedSpecId = newSpecId;
return newSpecId;
}

Expand Down Expand Up @@ -1722,13 +1713,7 @@ private int reuseOrCreateNewSpecId(PartitionSpec newSpec) {
private int addSortOrderInternal(SortOrder order) {
int newOrderId = reuseOrCreateNewSortOrderId(order);
if (sortOrdersById.containsKey(newOrderId)) {
// update lastAddedOrderId if the order was added in this set of changes (since it is now
// the last)
boolean isNewOrder =
lastAddedOrderId != null
&& changes(MetadataUpdate.AddSortOrder.class)
.anyMatch(added -> added.sortOrder().orderId() == lastAddedOrderId);
this.lastAddedOrderId = isNewOrder ? newOrderId : null;
this.lastAddedOrderId = newOrderId;
return newOrderId;
}

Expand Down Expand Up @@ -1920,6 +1905,18 @@ private boolean isAddedSnapshot(long snapshotId) {
.anyMatch(add -> add.snapshot().snapshotId() == snapshotId);
}

private <U extends MetadataUpdate> boolean isLastAddedChange(
Class<U> addClass, ToIntFunction<U> addedId, int id) {
for (int i = changes.size() - 1; i >= 0; i -= 1) {
MetadataUpdate change = changes.get(i);
if (addClass.isInstance(change)) {
return addedId.applyAsInt(addClass.cast(change)) == id;
}
}

return false;
}

private <U extends MetadataUpdate> Stream<U> changes(Class<U> updateClass) {
return changes.stream().filter(updateClass::isInstance).map(updateClass::cast);
}
Expand Down
98 changes: 98 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestTableMetadata.java
Original file line number Diff line number Diff line change
Expand Up @@ -1710,6 +1710,104 @@ schema, new Schema(1, schema2.columns()), new Schema(2, schema3.columns())),
assertThat(threeSchemaTable.lastColumnId()).isEqualTo(6);
}

@Test
void setCurrentSchemaToReAddedHistoricalSchema() {
Schema original = new Schema(Types.NestedField.required(1, "x", Types.LongType.get()));
TableMetadata base =
TableMetadata.newTableMetadata(
original, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of());
TableMetadata updated =
base.updateSchema(
new Schema(
Types.NestedField.required(1, "x", Types.LongType.get()),
Types.NestedField.optional(2, "y", Types.StringType.get())));

TableMetadata reverted =
TableMetadata.buildFrom(updated).addSchema(original).setCurrentSchema(-1).build();

assertThat(reverted.currentSchemaId()).isEqualTo(base.currentSchemaId());
assertThat(reverted.changes()).hasSize(updated.changes().size() + 1);
assertThat(Iterables.getLast(reverted.changes()))
.isInstanceOfSatisfying(
MetadataUpdate.SetCurrentSchema.class,
update -> assertThat(update.schemaId()).isEqualTo(base.currentSchemaId()));
}

@Test
void setDefaultSpecToReAddedHistoricalSpec() {
Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get()));
TableMetadata base =
TableMetadata.newTableMetadata(
schema, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of());
TableMetadata updated =
base.updatePartitionSpec(PartitionSpec.builderFor(base.schema()).bucket("x", 4).build());

TableMetadata reverted =
TableMetadata.buildFrom(updated)
.addPartitionSpec(PartitionSpec.unpartitioned())
.setDefaultPartitionSpec(-1)
.build();

assertThat(reverted.defaultSpecId()).isEqualTo(base.defaultSpecId());
assertThat(reverted.changes()).hasSize(updated.changes().size() + 1);
assertThat(Iterables.getLast(reverted.changes()))
.isInstanceOfSatisfying(
MetadataUpdate.SetDefaultPartitionSpec.class,
update -> assertThat(update.specId()).isEqualTo(base.defaultSpecId()));
}

@Test
void setDefaultSortOrderToReAddedHistoricalSortOrder() {
Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get()));
TableMetadata base =
TableMetadata.newTableMetadata(
schema, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of());
TableMetadata updated =
base.replaceSortOrder(SortOrder.builderFor(base.schema()).asc("x").build());

TableMetadata reverted =
TableMetadata.buildFrom(updated)
.addSortOrder(SortOrder.unsorted())
.setDefaultSortOrder(-1)
.build();

assertThat(reverted.defaultSortOrderId()).isEqualTo(base.defaultSortOrderId());
assertThat(reverted.changes()).hasSize(updated.changes().size() + 1);
assertThat(Iterables.getLast(reverted.changes()))
.isInstanceOfSatisfying(
MetadataUpdate.SetDefaultSortOrder.class,
update -> assertThat(update.sortOrderId()).isEqualTo(base.defaultSortOrderId()));
}

@Test
void replayedChangesSetCurrentSchemaToReAddedSchema() {

@jiwen624 jiwen624 Nov 11, 2026 •

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to cover schemas only. How about the specs and sort orders which are also changed?

Schema original = new Schema(Types.NestedField.required(1, "x", Types.LongType.get()));
TableMetadata base =
TableMetadata.newTableMetadata(
original, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of());
Schema first =
new Schema(
Types.NestedField.required(1, "x", Types.LongType.get()),
Types.NestedField.optional(2, "y", Types.StringType.get()));
Schema second =
new Schema(
Types.NestedField.required(1, "x", Types.LongType.get()),
Types.NestedField.optional(3, "z", Types.StringType.get()));

TableMetadata updated =
TableMetadata.buildFrom(base)
.addSchema(first)
.addSchema(second)
.addSchema(first)
.setCurrentSchema(-1)
.build();

TableMetadata.Builder replay = TableMetadata.buildFrom(base);
updated.changes().forEach(change -> change.applyTo(replay));

assertThat(replay.build().schema().asStruct()).isEqualTo(updated.schema().asStruct());
}

@Test
public void testCreateV2MetadataThroughTableProperty() {
Schema schema = new Schema(Types.NestedField.required(10, "x", Types.StringType.get()));
Expand Down
Loading