You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by yu...@apache.org on 2023/12/27 02:06:34 UTC

(incubator-paimon) branch release-0.6 updated: [core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)

This is an automated email from the ASF dual-hosted git repository.

yuzelin pushed a commit to branch release-0.6
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git


The following commit(s) were added to refs/heads/release-0.6 by this push:
     new 19aaeff3c [core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)
19aaeff3c is described below

commit 19aaeff3c0b47a130bb222591bfa707fe52cc869
Author: zqw142857 <13...@qq.com>
AuthorDate: Wed Dec 27 10:06:30 2023 +0800

    [core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)
    
    Co-authored-by: zhangqw152 <zh...@chinaunicom.cn>
    Co-authored-by: yuzelin <ze...@alibaba-inc.com>
---
 .../shortcodes/generated/core_configuration.html   |  6 ++
 .../main/java/org/apache/paimon/CoreOptions.java   | 13 ++++
 .../java/org/apache/paimon/AbstractFileStore.java  |  3 +-
 .../paimon/operation/FileStoreExpireImpl.java      |  9 ++-
 .../test/java/org/apache/paimon/TestFileStore.java | 11 +++-
 .../apache/paimon/operation/FileDeletionTest.java  | 76 ++++++++++++++++++++++
 6 files changed, 114 insertions(+), 4 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 82a8fc798..3b916d95a 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -497,6 +497,12 @@ This config option does not affect the default filesystem metastore.</td>
             <td>String</td>
             <td>The time zone to parse the long watermark value to TIMESTAMP value. The default value is 'UTC', which means the watermark is defined on TIMESTAMP column or not defined. If the watermark is defined on TIMESTAMP_LTZ column, the time zone of watermark is user configured time zone, the value should be the user configured local time zone. The option value is either a full name such as 'America/Los_Angeles', or a custom timezone id such as 'GMT-08:00'.</td>
         </tr>
+        <tr>
+            <td><h5>snapshot.expire.clean-empty-directories</h5></td>
+            <td style="word-wrap: break-word;">true</td>
+            <td>Boolean</td>
+            <td>Whether to try to clean empty directories when expiring snapshots. Note that trying to clean directories might throw exceptions in filesystem, but in most cases it won't cause problems.</td>
+        </tr>
         <tr>
             <td><h5>snapshot.expire.execution-mode</h5></td>
             <td style="word-wrap: break-word;">sync</td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 4274ea8d2..506e924e2 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -211,6 +211,15 @@ public class CoreOptions implements Serializable {
                     .withDescription(
                             "The maximum number of snapshots allowed to expire at a time.");
 
+    public static final ConfigOption<Boolean> SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES =
+            key("snapshot.expire.clean-empty-directories")
+                    .booleanType()
+                    .defaultValue(true)
+                    .withDescription(
+                            "Whether to try to clean empty directories when expiring snapshots. "
+                                    + "Note that trying to clean directories might throw exceptions in filesystem, "
+                                    + "but in most cases it won't cause problems.");
+
     public static final ConfigOption<Duration> CONTINUOUS_DISCOVERY_INTERVAL =
             key("continuous.discovery-interval")
                     .durationType()
@@ -1108,6 +1117,10 @@ public class CoreOptions implements Serializable {
         return options.get(SNAPSHOT_EXPIRE_LIMIT);
     }
 
+    public boolean snapshotExpireCleanEmptyDirectories() {
+        return options.get(SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES);
+    }
+
     public int manifestMergeMinCount() {
         return options.get(MANIFEST_MERGE_MIN_COUNT);
     }
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index cf6809b95..344118cd0 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -177,7 +177,8 @@ public abstract class AbstractFileStore<T> implements FileStore<T> {
                 snapshotManager(),
                 newSnapshotDeletion(),
                 newTagManager(),
-                options.snapshotExpireLimit());
+                options.snapshotExpireLimit(),
+                options.snapshotExpireCleanEmptyDirectories());
     }
 
     @Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
index ab6bb5d09..f6fcd87b1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
@@ -60,6 +60,7 @@ public class FileStoreExpireImpl implements FileStoreExpire {
 
     private final TagManager tagManager;
     private final int expireLimit;
+    private final boolean snapshotExpireCleanEmptyDirectories;
 
     private Lock lock;
 
@@ -70,7 +71,8 @@ public class FileStoreExpireImpl implements FileStoreExpire {
             SnapshotManager snapshotManager,
             SnapshotDeletion snapshotDeletion,
             TagManager tagManager,
-            int expireLimit) {
+            int expireLimit,
+            boolean snapshotExpireCleanEmptyDirectories) {
         Preconditions.checkArgument(
                 numRetainedMin >= 1,
                 "The minimum number of completed snapshots to retain should be >= 1.");
@@ -89,6 +91,7 @@ public class FileStoreExpireImpl implements FileStoreExpire {
         this.snapshotDeletion = snapshotDeletion;
         this.tagManager = tagManager;
         this.expireLimit = expireLimit;
+        this.snapshotExpireCleanEmptyDirectories = snapshotExpireCleanEmptyDirectories;
     }
 
     @Override
@@ -208,7 +211,9 @@ public class FileStoreExpireImpl implements FileStoreExpire {
 
         // data files and changelog files in bucket directories has been deleted
         // then delete changed bucket directories if they are empty
-        snapshotDeletion.cleanDataDirectories();
+        if (snapshotExpireCleanEmptyDirectories) {
+            snapshotDeletion.cleanDataDirectories();
+        }
 
         // delete manifests and indexFiles
         List<Snapshot> skippingSnapshots =
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index fbf058f7f..c8fc99432 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -133,6 +133,14 @@ public class TestFileStore extends KeyValueFileStore {
 
     public FileStoreExpireImpl newExpire(
             int numRetainedMin, int numRetainedMax, long millisRetained) {
+        return newExpire(numRetainedMin, numRetainedMax, millisRetained, true);
+    }
+
+    public FileStoreExpireImpl newExpire(
+            int numRetainedMin,
+            int numRetainedMax,
+            long millisRetained,
+            boolean snapshotExpireCleanEmptyDirectories) {
         return new FileStoreExpireImpl(
                 numRetainedMin,
                 numRetainedMax,
@@ -140,7 +148,8 @@ public class TestFileStore extends KeyValueFileStore {
                 snapshotManager(),
                 newSnapshotDeletion(),
                 new TagManager(fileIO, options.path()),
-                Integer.MAX_VALUE);
+                Integer.MAX_VALUE,
+                snapshotExpireCleanEmptyDirectories);
     }
 
     public List<Snapshot> commitData(
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
index d150ca457..30b28bd57 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
@@ -158,6 +158,82 @@ public class FileDeletionTest {
         assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
     }
 
+    /**
+     * This test checks FileStoreExpire can ignore clean data directories in snapshot expire
+     * situation. The partition keys are (dt, hr). Test process:
+     *
+     * <ul>
+     *   <li>1. Generate snapshot 1 with (0401, 8/12), (0402, 8/12). Each partition has two buckets.
+     *   <li>2. Generate snapshot 2 by deleting all data of partition dt=0401 (thus directory
+     *       dt=0401 will be exist after expiring).
+     *   <li>3. Generate snapshot 3 by deleting all data of partition dt=0402/hr=8 (thus directory
+     *       dt=0402/hr=8 will be exist after expiring).
+     *   <li>4. Generate snapshot 4 by deleting all data of partition dt=0402/hr=12/bucket-0 (thus
+     *       directory dt=0402/hr=12/bucket-0 will be exist after expiring).
+     *   <li>5. Expire snapshot 1-3 (dt=0402/hr=20/bucket-1 survives) and set ignore clean data
+     *       directories, then check.
+     * </ul>
+     */
+    @Test
+    public void testCleanDirectories() throws Exception {
+        TestFileStore store = createStore(TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED);
+        TestKeyValueGenerator gen =
+                new TestKeyValueGenerator(TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED);
+        FileStorePathFactory pathFactory = store.pathFactory();
+
+        // step 1: generate snapshot 1 by writing 5 randomly generated records to each bucket
+        // writers for each bucket
+        Map<BinaryRow, Map<Integer, RecordWriter<KeyValue>>> writers = new HashMap<>();
+
+        List<BinaryRow> partitions = new ArrayList<>();
+        for (String dt : Arrays.asList("0401", "0402")) {
+            for (int hr : Arrays.asList(8, 12)) {
+                for (int bucket : Arrays.asList(0, 1)) {
+                    List<KeyValue> kvs = partitionedData(5, gen, dt, hr);
+                    BinaryRow partition = gen.getPartition(kvs.get(0));
+                    partitions.add(partition);
+                    writeData(store, kvs, partition, bucket, writers);
+                }
+            }
+        }
+
+        commitData(store, commitIdentifier++, writers);
+        // check all paths exist
+        for (BinaryRow partition : partitions) {
+            for (int bucket : Arrays.asList(0, 1)) {
+                assertPathExists(fileIO, pathFactory.bucketPath(partition, bucket));
+            }
+        }
+
+        // step 2: generate snapshot 2 by cleaning partition dt=0401 (through overwriting with an
+        // empty ManifestCommittable)
+        FileStoreCommitImpl commit = store.newCommit();
+        Map<String, String> partitionSpec = new HashMap<>();
+        partitionSpec.put("dt", "0401");
+        commit.overwrite(
+                partitionSpec, new ManifestCommittable(commitIdentifier++), Collections.emptyMap());
+
+        // step 3: generate snapshot 3 by cleaning partition dt=0402/hr=8
+        partitionSpec.put("dt", "0402");
+        partitionSpec.put("hr", "8");
+        commit.overwrite(
+                partitionSpec, new ManifestCommittable(commitIdentifier++), Collections.emptyMap());
+
+        // step 4: generate snapshot 4 by cleaning dt=0402/hr=12/bucket-0
+        BinaryRow partition = partitions.get(7);
+        cleanBucket(store, partition, 0);
+
+        // step 5: expire and check file paths
+        store.newExpire(1, 1, Long.MAX_VALUE, false).expire();
+        // whole dt=0401 is exist
+        assertPathExists(fileIO, new Path(root, "dt=0401"));
+        // whole dt=0402/hr=8 is exist
+        assertPathExists(fileIO, new Path(root, "dt=0402/hr=8"));
+        // for dt=0402/hr=12, bucket-0 is exist and bucket-1 survives
+        assertPathExists(fileIO, pathFactory.bucketPath(partition, 0));
+        assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
+    }
+
     // only exists bucket directories
     @Test
     public void testNoPartitions() throws Exception {