Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 35 additions & 12 deletions core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
Expand Down Expand Up @@ -171,7 +172,7 @@ public List<ManifestFile> apply(TableMetadata base, Snapshot snapshot) {
List<ManifestFile> currentManifests = base.currentSnapshot().allManifests(ops().io());
Set<ManifestFile> currentManifestSet = ImmutableSet.copyOf(currentManifests);

validateDeletedManifests(currentManifestSet, base.currentSnapshot().snapshotId());
validateDeletedManifests(currentManifests, base.currentSnapshot().snapshotId());

if (requiresRewrite(currentManifestSet)) {
performRewrite(currentManifests);
Expand Down Expand Up @@ -284,17 +285,39 @@ private boolean matchesPredicate(ManifestFile manifest) {
}

private void validateDeletedManifests(
Set<ManifestFile> currentManifests, long currentSnapshotID) {
// directly deleted manifests must be still present in the current snapshot
deletedManifests.stream()
.filter(manifest -> !currentManifests.contains(manifest))
.findAny()
.ifPresent(
manifest -> {
throw new ValidationException(
"Deleted manifest %s could not be found in the latest snapshot %d",
manifest.path(), currentSnapshotID);
});
List<ManifestFile> currentManifests, long currentSnapshotID) {
Map<String, ManifestFile> currentManifestsByPath = Maps.newHashMap();
for (ManifestFile manifest : currentManifests) {
currentManifestsByPath.put(manifest.path(), manifest);
}

for (ManifestFile manifest : deletedManifests) {
// directly deleted manifests must be still present in the current snapshot
ManifestFile current = currentManifestsByPath.get(manifest.path());
ValidationException.check(
current != null,
"Deleted manifest %s could not be found in the latest snapshot %d",
manifest.path(),
currentSnapshotID);

// a manifest staged earlier in a transaction keeps its path when the transaction retries but
// can inherit a new sequence number and first row ID, which a replacement written from its
// entries would not have
ValidationException.check(
manifest.sequenceNumber() == current.sequenceNumber()
&& manifest.minSequenceNumber() == current.minSequenceNumber()
&& Objects.equals(manifest.firstRowId(), current.firstRowId()),
"Deleted manifest %s changed in the latest snapshot %d: "
+ "sequence number %s -> %s, min sequence number %s -> %s, first row ID %s -> %s",
manifest.path(),
currentSnapshotID,
manifest.sequenceNumber(),
current.sequenceNumber(),
manifest.minSequenceNumber(),
current.minSequenceNumber(),
manifest.firstRowId(),
current.firstRowId());
}
}

private void validateFilesCounts() {
Expand Down
76 changes: 76 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestRewriteManifests.java
Original file line number Diff line number Diff line change
Expand Up @@ -726,6 +726,82 @@ public void testManifestReplacementConcurrentAppend() throws IOException {
statuses(ManifestEntry.Status.ADDED, ManifestEntry.Status.ADDED));
}

@TestTemplate
public void testReplaceStagedManifestAfterTransactionRetry() throws IOException {
assumeThat(formatVersion).isGreaterThanOrEqualTo(2);

Transaction txn = table.newTransaction();
txn.newFastAppend().appendFile(FILE_A).commit();

Snapshot stagedAppend = txn.table().currentSnapshot();
List<ManifestFile> stagedManifests = stagedAppend.allManifests(table.io());
assertThat(stagedManifests).hasSize(1);
ManifestFile stagedManifest = stagedManifests.get(0);

// copy the staged manifest's entries with the metadata they inherit in this attempt
File replacementFile = temp.resolve(manifestFormat().addExtension("replacement")).toFile();
ManifestWriter<DataFile> writer =
ManifestFiles.write(
formatVersion,
table.spec(),
table.io().newOutputFile(replacementFile.getCanonicalPath()),
null);
try (ManifestReader<DataFile> reader =
ManifestFiles.read(stagedManifest, table.io(), table.specs())) {
for (ManifestEntry<DataFile> entry : reader.entries()) {
writer.existing(
entry.file(),
entry.snapshotId(),
entry.dataSequenceNumber(),
entry.fileSequenceNumber());
}
} finally {
writer.close();
}

txn.rewriteManifests()
.deleteManifest(stagedManifest)
.addManifest(writer.toManifestFile())
.commit();

// a concurrent commit forces the transaction to retry, which reuses the staged manifest
// but assigns it a new sequence number and first row ID
table.newFastAppend().appendFile(FILE_B).commit();
Snapshot concurrentAppend = table.currentSnapshot();

// committing the replacement would keep the sequence number and row IDs of the first attempt
assertThatThrownBy(txn::commitTransaction)
.isInstanceOf(ValidationException.class)
.hasMessageStartingWith(
String.format(
"Deleted manifest %s changed in the latest snapshot", stagedManifest.path()));

assertThat(table.currentSnapshot().snapshotId()).isEqualTo(concurrentAppend.snapshotId());
}

@TestTemplate
public void testReplaceCommittedManifestAfterTransactionRetry() throws IOException {
table.newFastAppend().appendFile(FILE_A).commit();
ManifestFile committedManifest = table.currentSnapshot().allManifests(table.io()).get(0);
ManifestFile replacement =
writeManifest(
"replacement.avro",
manifestEntry(
ManifestEntry.Status.EXISTING, table.currentSnapshot().snapshotId(), FILE_A));

Transaction txn = table.newTransaction();
txn.rewriteManifests().deleteManifest(committedManifest).addManifest(replacement).commit();

// a committed manifest keeps its inherited metadata when the transaction retries
table.newFastAppend().appendFile(FILE_B).commit();
txn.commitTransaction();

assertThat(table.currentSnapshot().allManifests(table.io()))
.extracting(ManifestFile::path)
.doesNotContain(committedManifest.path());
validateTableFiles(table, FILE_A, FILE_B);
}

@TestTemplate
public void testManifestReplacementConcurrentDelete() throws IOException {
assertThat(table.currentSnapshot()).isNull();
Expand Down
Loading