diff --git a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java index b0188b9854cf..0c8044ab13ec 100644 --- a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java +++ b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java @@ -26,6 +26,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; @@ -171,7 +172,7 @@ public List apply(TableMetadata base, Snapshot snapshot) { List currentManifests = base.currentSnapshot().allManifests(ops().io()); Set currentManifestSet = ImmutableSet.copyOf(currentManifests); - validateDeletedManifests(currentManifestSet, base.currentSnapshot().snapshotId()); + validateDeletedManifests(currentManifests, base.currentSnapshot().snapshotId()); if (requiresRewrite(currentManifestSet)) { performRewrite(currentManifests); @@ -284,17 +285,39 @@ private boolean matchesPredicate(ManifestFile manifest) { } private void validateDeletedManifests( - Set currentManifests, long currentSnapshotID) { - // directly deleted manifests must be still present in the current snapshot - deletedManifests.stream() - .filter(manifest -> !currentManifests.contains(manifest)) - .findAny() - .ifPresent( - manifest -> { - throw new ValidationException( - "Deleted manifest %s could not be found in the latest snapshot %d", - manifest.path(), currentSnapshotID); - }); + List currentManifests, long currentSnapshotID) { + Map currentManifestsByPath = Maps.newHashMap(); + for (ManifestFile manifest : currentManifests) { + currentManifestsByPath.put(manifest.path(), manifest); + } + + for (ManifestFile manifest : deletedManifests) { + // directly deleted manifests must be still present in the current snapshot + ManifestFile current = currentManifestsByPath.get(manifest.path()); + ValidationException.check( + current != null, + "Deleted manifest %s could not be found in the latest snapshot %d", + manifest.path(), + currentSnapshotID); + + // a manifest staged earlier in a transaction keeps its path when the transaction retries but + // can inherit a new sequence number and first row ID, which a replacement written from its + // entries would not have + ValidationException.check( + manifest.sequenceNumber() == current.sequenceNumber() + && manifest.minSequenceNumber() == current.minSequenceNumber() + && Objects.equals(manifest.firstRowId(), current.firstRowId()), + "Deleted manifest %s changed in the latest snapshot %d: " + + "sequence number %s -> %s, min sequence number %s -> %s, first row ID %s -> %s", + manifest.path(), + currentSnapshotID, + manifest.sequenceNumber(), + current.sequenceNumber(), + manifest.minSequenceNumber(), + current.minSequenceNumber(), + manifest.firstRowId(), + current.firstRowId()); + } } private void validateFilesCounts() { diff --git a/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java b/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java index dab323743bb1..e2c61775800d 100644 --- a/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java +++ b/core/src/test/java/org/apache/iceberg/TestRewriteManifests.java @@ -726,6 +726,82 @@ public void testManifestReplacementConcurrentAppend() throws IOException { statuses(ManifestEntry.Status.ADDED, ManifestEntry.Status.ADDED)); } + @TestTemplate + public void testReplaceStagedManifestAfterTransactionRetry() throws IOException { + assumeThat(formatVersion).isGreaterThanOrEqualTo(2); + + Transaction txn = table.newTransaction(); + txn.newFastAppend().appendFile(FILE_A).commit(); + + Snapshot stagedAppend = txn.table().currentSnapshot(); + List stagedManifests = stagedAppend.allManifests(table.io()); + assertThat(stagedManifests).hasSize(1); + ManifestFile stagedManifest = stagedManifests.get(0); + + // copy the staged manifest's entries with the metadata they inherit in this attempt + File replacementFile = temp.resolve(manifestFormat().addExtension("replacement")).toFile(); + ManifestWriter writer = + ManifestFiles.write( + formatVersion, + table.spec(), + table.io().newOutputFile(replacementFile.getCanonicalPath()), + null); + try (ManifestReader reader = + ManifestFiles.read(stagedManifest, table.io(), table.specs())) { + for (ManifestEntry entry : reader.entries()) { + writer.existing( + entry.file(), + entry.snapshotId(), + entry.dataSequenceNumber(), + entry.fileSequenceNumber()); + } + } finally { + writer.close(); + } + + txn.rewriteManifests() + .deleteManifest(stagedManifest) + .addManifest(writer.toManifestFile()) + .commit(); + + // a concurrent commit forces the transaction to retry, which reuses the staged manifest + // but assigns it a new sequence number and first row ID + table.newFastAppend().appendFile(FILE_B).commit(); + Snapshot concurrentAppend = table.currentSnapshot(); + + // committing the replacement would keep the sequence number and row IDs of the first attempt + assertThatThrownBy(txn::commitTransaction) + .isInstanceOf(ValidationException.class) + .hasMessageStartingWith( + String.format( + "Deleted manifest %s changed in the latest snapshot", stagedManifest.path())); + + assertThat(table.currentSnapshot().snapshotId()).isEqualTo(concurrentAppend.snapshotId()); + } + + @TestTemplate + public void testReplaceCommittedManifestAfterTransactionRetry() throws IOException { + table.newFastAppend().appendFile(FILE_A).commit(); + ManifestFile committedManifest = table.currentSnapshot().allManifests(table.io()).get(0); + ManifestFile replacement = + writeManifest( + "replacement.avro", + manifestEntry( + ManifestEntry.Status.EXISTING, table.currentSnapshot().snapshotId(), FILE_A)); + + Transaction txn = table.newTransaction(); + txn.rewriteManifests().deleteManifest(committedManifest).addManifest(replacement).commit(); + + // a committed manifest keeps its inherited metadata when the transaction retries + table.newFastAppend().appendFile(FILE_B).commit(); + txn.commitTransaction(); + + assertThat(table.currentSnapshot().allManifests(table.io())) + .extracting(ManifestFile::path) + .doesNotContain(committedManifest.path()); + validateTableFiles(table, FILE_A, FILE_B); + } + @TestTemplate public void testManifestReplacementConcurrentDelete() throws IOException { assertThat(table.currentSnapshot()).isNull();