[incubator-iceberg] branch master updated: Delete manifest lists after expiring snapshots. (#102)

[bl...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 460d080  Delete manifest lists after expiring snapshots. (#102)
460d080 is described below

commit 460d08099fb285e184109b2cfcea9b50c0c5c8ce
Author: Ryan Blue <rd...@users.noreply.github.com>
AuthorDate: Tue Mar 5 16:13:27 2019 -0800

    Delete manifest lists after expiring snapshots. (#102)
---
 .../main/java/com/netflix/iceberg/RemoveSnapshots.java   | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/core/src/main/java/com/netflix/iceberg/RemoveSnapshots.java b/core/src/main/java/com/netflix/iceberg/RemoveSnapshots.java
index 4784dd1..5c16e49 100644
--- a/core/src/main/java/com/netflix/iceberg/RemoveSnapshots.java
+++ b/core/src/main/java/com/netflix/iceberg/RemoveSnapshots.java
@@ -127,6 +127,7 @@ class RemoveSnapshots implements ExpireSnapshots {
     // 1. Get a list of the snapshots that were removed
     // 2. Delete any data files that were deleted by those snapshots and are not in the table
     // 3. Delete any manifests that are no longer used by current snapshots
+    // 4. Delete the manifest lists
 
     // Reads and deletes are done using Tasks.foreach(...).suppressFailureWhenFinished to complete
     // as much of the delete work as possible and avoid orphaned data or manifest files.
@@ -139,19 +140,25 @@ class RemoveSnapshots implements ExpireSnapshots {
       currentManifests.addAll(snapshot.manifests());
     }
 
+    Set<String> manifestListsToDelete = Sets.newHashSet();
     Set<ManifestFile> allManifests = Sets.newHashSet(currentManifests);
     Set<String> manifestsToDelete = Sets.newHashSet();
     for (Snapshot snapshot : base.snapshots()) {
       long snapshotId = snapshot.snapshotId();
       if (!currentIds.contains(snapshotId)) {
-        // the snapshot was removed, find any manifests that are no longer needed
-        LOG.info("Removing snapshot: {}", snapshot);
+        // the snapshot was expired
+        LOG.info("Expired snapshot: {}", snapshot);
+        // find any manifests that are no longer needed
         for (ManifestFile manifest : snapshot.manifests()) {
           if (!currentManifests.contains(manifest)) {
             manifestsToDelete.add(manifest.path());
             allManifests.add(manifest);
           }
         }
+        // add the manifest list to the delete set, if present
+        if (snapshot.manifestListLocation() != null) {
+          manifestListsToDelete.add(snapshot.manifestListLocation());
+        }
       }
     }
 
@@ -192,5 +199,10 @@ class RemoveSnapshots implements ExpireSnapshots {
         .noRetry().suppressFailureWhenFinished()
         .onFailure((manifest, exc) -> LOG.warn("Delete failed for manifest: " + manifest, exc))
         .run(deleteFunc::accept);
+
+    Tasks.foreach(manifestListsToDelete)
+        .noRetry().suppressFailureWhenFinished()
+        .onFailure((list, exc) -> LOG.warn("Delete failed for manifest list: " + list, exc))
+        .run(deleteFunc::accept);
   }
 }