Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 63 additions & 2 deletions core/src/main/java/org/apache/iceberg/IncrementalFileCleanup.java
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public void cleanFiles(
// find manifests to clean up that are still referenced by a valid snapshot, but written by an
// expired snapshot
Set<String> validManifests = ConcurrentHashMap.newKeySet();
Set<ManifestFile> liveDeleteManifests = ConcurrentHashMap.newKeySet();
Set<ManifestFile> manifestsToScan = ConcurrentHashMap.newKeySet();

// Reads and deletes are done using Tasks.foreach(...).suppressFailureWhenFinished to complete
Expand All @@ -131,6 +132,9 @@ public void cleanFiles(
try (CloseableIterable<ManifestFile> manifests = readManifests(snapshot)) {
for (ManifestFile manifest : manifests) {
validManifests.add(manifest.path());
if (manifest.content() == ManifestContent.DELETES) {
liveDeleteManifests.add(manifest.copy());
}

long snapshotId = manifest.snapshotId();
// whether the manifest was created by a valid snapshot (true) or an expired
Expand Down Expand Up @@ -261,7 +265,11 @@ public void cleanFiles(
if (ExpireSnapshots.CleanupLevel.ALL == cleanupLevel) {
Set<String> filesToDelete =
findFilesToDelete(
manifestsToScan, manifestsToRevert, validIds, beforeExpiration.specsById());
manifestsToScan,
manifestsToRevert,
liveDeleteManifests,
validIds,
beforeExpiration.specsById());
LOG.debug("Deleting {} data files", filesToDelete.size());
deleteFiles(filesToDelete, "data");
}
Expand All @@ -282,9 +290,11 @@ public void cleanFiles(
private Set<String> findFilesToDelete(
Set<ManifestFile> manifestsToScan,
Set<ManifestFile> manifestsToRevert,
Set<ManifestFile> liveDeleteManifests,
Set<Long> validIds,
Map<Integer, PartitionSpec> specsById) {
Set<String> filesToDelete = ConcurrentHashMap.newKeySet();
Set<String> dvFileRemovalCandidates = ConcurrentHashMap.newKeySet();
Tasks.foreach(manifestsToScan)
.retry(3)
.suppressFailureWhenFinished()
Expand All @@ -302,14 +312,21 @@ private Set<String> findFilesToDelete(
if (entry.status() == ManifestEntry.Status.DELETED
&& !validIds.contains(entry.snapshotId())) {
// use toString to ensure the path will not change (Utf8 is reused)
filesToDelete.add(entry.file().location());
String location = entry.file().location();
if (entry.file().format() == FileFormat.PUFFIN) {
dvFileRemovalCandidates.add(location);
} else {
filesToDelete.add(location);
}
}
}
} catch (IOException e) {
throw new RuntimeIOException(e, "Failed to read manifest file: %s", manifest);
}
});

filesToDelete.addAll(unreferencedDVs(dvFileRemovalCandidates, liveDeleteManifests, specsById));

Tasks.foreach(manifestsToRevert)
.retry(3)
.suppressFailureWhenFinished()
Expand All @@ -335,4 +352,48 @@ private Set<String> findFilesToDelete(

return filesToDelete;
}

// A Puffin file can be shared by multiple deletion vectors, so keep it if any live one remains
private Set<String> unreferencedDVs(
Set<String> dvFileRemovalCandidates,
Set<ManifestFile> liveDeleteManifests,
Map<Integer, PartitionSpec> specsById) {
if (dvFileRemovalCandidates.isEmpty()) {
return dvFileRemovalCandidates;
}

Set<String> unreferenced = ConcurrentHashMap.newKeySet();
unreferenced.addAll(dvFileRemovalCandidates);

try {
Tasks.foreach(liveDeleteManifests)
.retry(3)
.stopOnFailure()
.throwFailureWhenFinished()
.executeWith(planExecutorService)
.onFailure(
(item, exc) ->
LOG.warn("Failed to determine live DVs in manifest {}", item.path(), exc))
.run(
manifest -> {
if (unreferenced.isEmpty()) {
return;
}

try (CloseableIterable<String> paths =
ManifestFiles.readPaths(manifest, fileIO, specsById)) {
for (String path : paths) {
unreferenced.remove(path);
}
} catch (IOException e) {
throw new RuntimeIOException(e, "Failed to read manifest file: %s", manifest);
}
});
} catch (Throwable e) {
LOG.warn("Failed to determine live DVs, skipping DV cleanup", e);
return Set.of();
}

return unreferenced;
}
}
83 changes: 83 additions & 0 deletions core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
Original file line number Diff line number Diff line change
Expand Up @@ -2144,6 +2144,89 @@ public void testCleanupLevelNullValidation() {
.hasMessageContaining("Invalid cleanup level: null");
}

@TestTemplate
void expireDoesNotDeletePuffinFileSharedByLiveDeletionVectors() {
assumeThat(formatVersion).as("Deletion vectors require V3 or later").isGreaterThanOrEqualTo(3);

table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();

// one Puffin file with deletion vectors for FILE_A and FILE_B
String sharedPuffinPath = "/path/to/shared-dvs-" + UUID.randomUUID() + ".puffin";
DeleteFile dvA = deletionVector(sharedPuffinPath, FILE_A, 4L);
DeleteFile dvB = deletionVector(sharedPuffinPath, FILE_B, 44L);
table.newRowDelta().addDeletes(dvA).addDeletes(dvB).commit();

// replace FILE_A's deletion vector; FILE_B's stays in the shared Puffin file
DeleteFile newDvA =
deletionVector("/path/to/new-dv-" + UUID.randomUUID() + ".puffin", FILE_A, 4L);
table
.newRowDelta()
.validateFromSnapshot(table.currentSnapshot().snapshotId())
.removeDeletes(dvA)
.addDeletes(newDvA)
.commit();

table.newAppend().appendFile(FILE_C).commit();

long tAfterCommits = waitUntilAfter(table.currentSnapshot().timestampMillis());

Set<String> deletedFiles = Sets.newHashSet();
removeSnapshots(table).expireOlderThan(tAfterCommits).deleteWith(deletedFiles::add).commit();

assertThat(table.snapshots()).hasSize(1);
assertThat(deletedFiles)
.as("Puffin file with a live deletion vector must not be deleted")
.doesNotContain(sharedPuffinPath);
}

@TestTemplate
void expireDeletesPuffinFileWhenAllSharedDeletionVectorsReplaced() {
assumeThat(formatVersion).as("Deletion vectors require V3 or later").isGreaterThanOrEqualTo(3);

table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit();

String sharedPuffinPath = "/path/to/shared-dvs-" + UUID.randomUUID() + ".puffin";
DeleteFile dvA = deletionVector(sharedPuffinPath, FILE_A, 4L);
DeleteFile dvB = deletionVector(sharedPuffinPath, FILE_B, 44L);
table.newRowDelta().addDeletes(dvA).addDeletes(dvB).commit();

DeleteFile newDvA =
deletionVector("/path/to/new-dv-a-" + UUID.randomUUID() + ".puffin", FILE_A, 4L);
DeleteFile newDvB =
deletionVector("/path/to/new-dv-b-" + UUID.randomUUID() + ".puffin", FILE_B, 4L);
table
.newRowDelta()
.validateFromSnapshot(table.currentSnapshot().snapshotId())
.removeDeletes(dvA)
.removeDeletes(dvB)
.addDeletes(newDvA)
.addDeletes(newDvB)
.commit();

table.newAppend().appendFile(FILE_C).commit();

long tAfterCommits = waitUntilAfter(table.currentSnapshot().timestampMillis());

Set<String> deletedFiles = Sets.newHashSet();
removeSnapshots(table).expireOlderThan(tAfterCommits).deleteWith(deletedFiles::add).commit();

assertThat(table.snapshots()).hasSize(1);
assertThat(deletedFiles).contains(sharedPuffinPath);
}

private DeleteFile deletionVector(String puffinPath, DataFile dataFile, long contentOffset) {
return FileMetadata.deleteFileBuilder(table.spec())
.ofPositionDeletes()
.withPath(puffinPath)
.withFileSizeInBytes(100L)
.withPartition(dataFile.partition())
.withRecordCount(1L)
.withReferencedDataFile(dataFile.location())
.withContentOffset(contentOffset)
.withContentSizeInBytes(40L)
.build();
}

@TestTemplate
void readManifestsProjectsManifestContent() throws IOException {
assumeThat(formatVersion)
Expand Down
Loading