diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 89427e4c6943..5af43255986d 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -1167,6 +1167,7 @@ Builder removeSpecs(Iterable specIds) { specs.stream() .filter(s -> !specIdsToRemove.contains(s.specId())) .collect(Collectors.toList()); + specsById.keySet().removeAll(specIdsToRemove); changes.add(new MetadataUpdate.RemovePartitionSpecs(specIdsToRemove)); } @@ -1183,6 +1184,7 @@ Builder removeSchemas(Iterable schemaIds) { schemas.stream() .filter(s -> !schemaIdsToRemove.contains(s.schemaId())) .collect(Collectors.toList()); + schemasById.keySet().removeAll(schemaIdsToRemove); changes.add(new MetadataUpdate.RemoveSchemas(schemaIdsToRemove)); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 02c23405033a..c13b05cd23cd 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -2071,6 +2071,74 @@ public void testMetadataWithRemoveSchemas() { assertThat(meta.changes()).anyMatch(u -> u instanceof MetadataUpdate.RemoveSchemas); } + @Test + public void testRemoveSchemasThenReuseRemovedId() { + TableMetadata base = + TableMetadata.buildFrom( + TableMetadata.newTableMetadata( + TEST_SCHEMA, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of())) + .addSchema( + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.optional(4, "unused", Types.LongType.get()))) + .build(); + assertThat(base.currentSchemaId()).isEqualTo(0); + assertThat(base.schemasById()).containsOnlyKeys(0, 1); + + assertThatThrownBy( + () -> + TableMetadata.buildFrom(base).removeSchemas(Sets.newHashSet(1)).setCurrentSchema(1)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot set current schema to unknown schema: 1"); + + // the new schema is assigned the id of the removed schema + Schema newSchema = + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.required(2, "y", Types.LongType.get(), "comment"), + Types.NestedField.required(3, "z", Types.LongType.get()), + Types.NestedField.optional(5, "w", Types.LongType.get())); + TableMetadata updated = + TableMetadata.buildFrom(base) + .removeSchemas(Sets.newHashSet(1)) + .setCurrentSchema(newSchema, 5) + .build(); + + assertThat(updated.schemasById()).containsOnlyKeys(0, 1); + assertThat(updated.currentSchemaId()).isEqualTo(1); + assertThat(updated.schema().asStruct()).isEqualTo(newSchema.asStruct()); + } + + @Test + public void testRemoveSpecsThenReuseRemovedId() { + TableMetadata created = + TableMetadata.newTableMetadata( + TEST_SCHEMA, PartitionSpec.unpartitioned(), TEST_LOCATION, ImmutableMap.of()); + TableMetadata base = + TableMetadata.buildFrom(created) + .addPartitionSpec(PartitionSpec.builderFor(created.schema()).bucket("y", 16).build()) + .build(); + assertThat(base.defaultSpecId()).isEqualTo(0); + assertThat(base.specsById()).containsOnlyKeys(0, 1); + + // the new spec is assigned the id of the removed spec + PartitionSpec newSpec = PartitionSpec.builderFor(base.schema()).identity("x").build(); + TableMetadata updated = + TableMetadata.buildFrom(base) + .removeSpecs(Sets.newHashSet(1)) + .setDefaultPartitionSpec(newSpec) + .build(); + + assertThat(updated.specsById()).containsOnlyKeys(0, 1); + assertThat(updated.defaultSpecId()).isEqualTo(1); + assertThat(updated.spec()) + .isEqualTo( + PartitionSpec.builderFor(updated.schema()) + .withSpecId(1) + .add(1, 1000, "x", Transforms.identity()) + .build()); + } + @Test public void testAddSnapshotWithStaleSequenceNumberIsRetryable() { TableMetadata base = diff --git a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java index 9e6ad888c601..2ddd48acae55 100644 --- a/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java +++ b/core/src/test/java/org/apache/iceberg/catalog/CatalogTests.java @@ -2013,6 +2013,72 @@ public void testRemoveUnusedSchemas(boolean withBranch) { } } + @Test + public void testRemoveUnusedSchemaThenAddColumnInTransaction() { + C catalog = catalog(); + + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + + Table table = + catalog.buildTable(TABLE, SCHEMA).withProperty(TableProperties.GC_ENABLED, "true").create(); + + // the current schema goes back to id 0 and schema 1 is no longer used + table.updateSchema().addColumn("col_to_delete", Types.IntegerType.get()).commit(); + table.updateSchema().deleteColumn("col_to_delete").commit(); + assertThat(table.schema().schemaId()).isEqualTo(0); + assertThat(table.schemas().keySet()).containsExactlyInAnyOrder(0, 1); + + // schema 1 is removed and the new schema is assigned the freed id + Transaction transaction = table.newTransaction(); + transaction.expireSnapshots().cleanExpiredMetadata(true).commit(); + transaction.updateSchema().addColumn("new_col", Types.LongType.get()).commit(); + transaction.commitTransaction(); + + Table loaded = catalog.loadTable(TABLE); + assertThat(loaded.schema().schemaId()).isEqualTo(1); + assertThat(loaded.schema().findField("new_col")).isNotNull(); + assertThat(loaded.schema().findField("col_to_delete")).isNull(); + assertThat(loaded.schemas().keySet()).containsExactlyInAnyOrder(0, 1); + } + + @Test + public void testRemoveUnusedSpecThenAddFieldInTransaction() { + C catalog = catalog(); + + if (requiresNamespaceCreate()) { + catalog.createNamespace(NS); + } + + // create a v2 table. otherwise removing the field would produce a new spec with a void + // partition field instead of going back to spec 0 + Table table = + catalog + .buildTable(TABLE, SCHEMA) + .withProperty(TableProperties.GC_ENABLED, "true") + .withProperty("format-version", "2") + .create(); + + // the default spec goes back to id 0 and spec 1 is no longer used + table.updateSpec().addField(Expressions.bucket("data", 16)).commit(); + table.updateSpec().removeField(Expressions.bucket("data", 16)).commit(); + assertThat(table.spec().specId()).isEqualTo(0); + assertThat(table.specs().keySet()).containsExactlyInAnyOrder(0, 1); + + // spec 1 is removed and the new spec is assigned the freed id + Transaction transaction = table.newTransaction(); + transaction.expireSnapshots().cleanExpiredMetadata(true).commit(); + transaction.updateSpec().addField("id").commit(); + transaction.commitTransaction(); + + Table loaded = catalog.loadTable(TABLE); + assertThat(loaded.spec().specId()).isEqualTo(1); + assertThat(loaded.spec().fields()).hasSize(1); + assertThat(loaded.spec().fields().get(0).name()).isEqualTo("id"); + assertThat(loaded.specs().keySet()).containsExactlyInAnyOrder(0, 1); + } + @Test public void testUpdateTableSortOrder() { C catalog = catalog();