From 9d15908f6013d0b055c2527ea2b850d3e87062c0 Mon Sep 17 00:00:00 2001 From: Yuya Ebihara Date: Wed, 14 Oct 2026 11:59:42 +0900 Subject: [PATCH] Core: Fix file cleanup logic to retain Puffin files shared by live deletion vectors (#18408) (cherry picked from commit c131e75958ff7bef67e22d241b02e5f8338b3955) --- .../iceberg/IncrementalFileCleanup.java | 65 ++++++++++++++- .../apache/iceberg/TestRemoveSnapshots.java | 83 +++++++++++++++++++ 2 files changed, 146 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/IncrementalFileCleanup.java b/core/src/main/java/org/apache/iceberg/IncrementalFileCleanup.java index 911a94c13f28..73a1e4584f67 100644 --- a/core/src/main/java/org/apache/iceberg/IncrementalFileCleanup.java +++ b/core/src/main/java/org/apache/iceberg/IncrementalFileCleanup.java @@ -111,6 +111,7 @@ public void cleanFiles( // find manifests to clean up that are still referenced by a valid snapshot, but written by an // expired snapshot Set validManifests = ConcurrentHashMap.newKeySet(); + Set liveDeleteManifests = ConcurrentHashMap.newKeySet(); Set manifestsToScan = ConcurrentHashMap.newKeySet(); // Reads and deletes are done using Tasks.foreach(...).suppressFailureWhenFinished to complete @@ -131,6 +132,9 @@ public void cleanFiles( try (CloseableIterable manifests = readManifests(snapshot)) { for (ManifestFile manifest : manifests) { validManifests.add(manifest.path()); + if (manifest.content() == ManifestContent.DELETES) { + liveDeleteManifests.add(manifest.copy()); + } long snapshotId = manifest.snapshotId(); // whether the manifest was created by a valid snapshot (true) or an expired @@ -261,7 +265,11 @@ public void cleanFiles( if (ExpireSnapshots.CleanupLevel.ALL == cleanupLevel) { Set filesToDelete = findFilesToDelete( - manifestsToScan, manifestsToRevert, validIds, beforeExpiration.specsById()); + manifestsToScan, + manifestsToRevert, + liveDeleteManifests, + validIds, + beforeExpiration.specsById()); LOG.debug("Deleting {} data files", filesToDelete.size()); deleteFiles(filesToDelete, "data"); } @@ -282,9 +290,11 @@ public void cleanFiles( private Set findFilesToDelete( Set manifestsToScan, Set manifestsToRevert, + Set liveDeleteManifests, Set validIds, Map specsById) { Set filesToDelete = ConcurrentHashMap.newKeySet(); + Set dvFileRemovalCandidates = ConcurrentHashMap.newKeySet(); Tasks.foreach(manifestsToScan) .retry(3) .suppressFailureWhenFinished() @@ -302,7 +312,12 @@ private Set findFilesToDelete( if (entry.status() == ManifestEntry.Status.DELETED && !validIds.contains(entry.snapshotId())) { // use toString to ensure the path will not change (Utf8 is reused) - filesToDelete.add(entry.file().location()); + String location = entry.file().location(); + if (entry.file().format() == FileFormat.PUFFIN) { + dvFileRemovalCandidates.add(location); + } else { + filesToDelete.add(location); + } } } } catch (IOException e) { @@ -310,6 +325,8 @@ private Set findFilesToDelete( } }); + filesToDelete.addAll(unreferencedDVs(dvFileRemovalCandidates, liveDeleteManifests, specsById)); + Tasks.foreach(manifestsToRevert) .retry(3) .suppressFailureWhenFinished() @@ -335,4 +352,48 @@ private Set findFilesToDelete( return filesToDelete; } + + // A Puffin file can be shared by multiple deletion vectors, so keep it if any live one remains + private Set unreferencedDVs( + Set dvFileRemovalCandidates, + Set liveDeleteManifests, + Map specsById) { + if (dvFileRemovalCandidates.isEmpty()) { + return dvFileRemovalCandidates; + } + + Set unreferenced = ConcurrentHashMap.newKeySet(); + unreferenced.addAll(dvFileRemovalCandidates); + + try { + Tasks.foreach(liveDeleteManifests) + .retry(3) + .stopOnFailure() + .throwFailureWhenFinished() + .executeWith(planExecutorService) + .onFailure( + (item, exc) -> + LOG.warn("Failed to determine live DVs in manifest {}", item.path(), exc)) + .run( + manifest -> { + if (unreferenced.isEmpty()) { + return; + } + + try (CloseableIterable paths = + ManifestFiles.readPaths(manifest, fileIO, specsById)) { + for (String path : paths) { + unreferenced.remove(path); + } + } catch (IOException e) { + throw new RuntimeIOException(e, "Failed to read manifest file: %s", manifest); + } + }); + } catch (Throwable e) { + LOG.warn("Failed to determine live DVs, skipping DV cleanup", e); + return Set.of(); + } + + return unreferenced; + } } diff --git a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java index 7a983bf58563..e3dd980dfeec 100644 --- a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java +++ b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java @@ -2144,6 +2144,89 @@ public void testCleanupLevelNullValidation() { .hasMessageContaining("Invalid cleanup level: null"); } + @TestTemplate + void expireDoesNotDeletePuffinFileSharedByLiveDeletionVectors() { + assumeThat(formatVersion).as("Deletion vectors require V3 or later").isGreaterThanOrEqualTo(3); + + table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); + + // one Puffin file with deletion vectors for FILE_A and FILE_B + String sharedPuffinPath = "/path/to/shared-dvs-" + UUID.randomUUID() + ".puffin"; + DeleteFile dvA = deletionVector(sharedPuffinPath, FILE_A, 4L); + DeleteFile dvB = deletionVector(sharedPuffinPath, FILE_B, 44L); + table.newRowDelta().addDeletes(dvA).addDeletes(dvB).commit(); + + // replace FILE_A's deletion vector; FILE_B's stays in the shared Puffin file + DeleteFile newDvA = + deletionVector("/path/to/new-dv-" + UUID.randomUUID() + ".puffin", FILE_A, 4L); + table + .newRowDelta() + .validateFromSnapshot(table.currentSnapshot().snapshotId()) + .removeDeletes(dvA) + .addDeletes(newDvA) + .commit(); + + table.newAppend().appendFile(FILE_C).commit(); + + long tAfterCommits = waitUntilAfter(table.currentSnapshot().timestampMillis()); + + Set deletedFiles = Sets.newHashSet(); + removeSnapshots(table).expireOlderThan(tAfterCommits).deleteWith(deletedFiles::add).commit(); + + assertThat(table.snapshots()).hasSize(1); + assertThat(deletedFiles) + .as("Puffin file with a live deletion vector must not be deleted") + .doesNotContain(sharedPuffinPath); + } + + @TestTemplate + void expireDeletesPuffinFileWhenAllSharedDeletionVectorsReplaced() { + assumeThat(formatVersion).as("Deletion vectors require V3 or later").isGreaterThanOrEqualTo(3); + + table.newAppend().appendFile(FILE_A).appendFile(FILE_B).commit(); + + String sharedPuffinPath = "/path/to/shared-dvs-" + UUID.randomUUID() + ".puffin"; + DeleteFile dvA = deletionVector(sharedPuffinPath, FILE_A, 4L); + DeleteFile dvB = deletionVector(sharedPuffinPath, FILE_B, 44L); + table.newRowDelta().addDeletes(dvA).addDeletes(dvB).commit(); + + DeleteFile newDvA = + deletionVector("/path/to/new-dv-a-" + UUID.randomUUID() + ".puffin", FILE_A, 4L); + DeleteFile newDvB = + deletionVector("/path/to/new-dv-b-" + UUID.randomUUID() + ".puffin", FILE_B, 4L); + table + .newRowDelta() + .validateFromSnapshot(table.currentSnapshot().snapshotId()) + .removeDeletes(dvA) + .removeDeletes(dvB) + .addDeletes(newDvA) + .addDeletes(newDvB) + .commit(); + + table.newAppend().appendFile(FILE_C).commit(); + + long tAfterCommits = waitUntilAfter(table.currentSnapshot().timestampMillis()); + + Set deletedFiles = Sets.newHashSet(); + removeSnapshots(table).expireOlderThan(tAfterCommits).deleteWith(deletedFiles::add).commit(); + + assertThat(table.snapshots()).hasSize(1); + assertThat(deletedFiles).contains(sharedPuffinPath); + } + + private DeleteFile deletionVector(String puffinPath, DataFile dataFile, long contentOffset) { + return FileMetadata.deleteFileBuilder(table.spec()) + .ofPositionDeletes() + .withPath(puffinPath) + .withFileSizeInBytes(100L) + .withPartition(dataFile.partition()) + .withRecordCount(1L) + .withReferencedDataFile(dataFile.location()) + .withContentOffset(contentOffset) + .withContentSizeInBytes(40L) + .build(); + } + @TestTemplate void readManifestsProjectsManifestContent() throws IOException { assumeThat(formatVersion)