You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by yu...@apache.org on 2023/12/27 07:35:40 UTC
(incubator-paimon) branch master updated: [core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)
This is an automated email from the ASF dual-hosted git repository.
yuzelin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 9d4b12896 [core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)
9d4b12896 is described below
commit 9d4b1289654ee628c489713fa4ba791c456e366a
Author: zqw142857 <13...@qq.com>
AuthorDate: Wed Dec 27 10:06:30 2023 +0800
[core] Add "snapshot.expire.clean-empty-directories" to determine whether to clean empty directories when expiring snapshots (#2522)
Co-authored-by: zhangqw152 <zh...@chinaunicom.cn>
Co-authored-by: yuzelin <ze...@alibaba-inc.com>
---
.../shortcodes/generated/core_configuration.html | 6 ++
.../main/java/org/apache/paimon/CoreOptions.java | 13 ++++
.../java/org/apache/paimon/AbstractFileStore.java | 3 +-
.../paimon/operation/FileStoreExpireImpl.java | 9 ++-
.../test/java/org/apache/paimon/TestFileStore.java | 11 +++-
.../apache/paimon/operation/FileDeletionTest.java | 76 ++++++++++++++++++++++
6 files changed, 114 insertions(+), 4 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/core_configuration.html b/docs/layouts/shortcodes/generated/core_configuration.html
index 1ae196721..16d6376ef 100644
--- a/docs/layouts/shortcodes/generated/core_configuration.html
+++ b/docs/layouts/shortcodes/generated/core_configuration.html
@@ -509,6 +509,12 @@ This config option does not affect the default filesystem metastore.</td>
<td>String</td>
<td>The time zone to parse the long watermark value to TIMESTAMP value. The default value is 'UTC', which means the watermark is defined on TIMESTAMP column or not defined. If the watermark is defined on TIMESTAMP_LTZ column, the time zone of watermark is user configured time zone, the value should be the user configured local time zone. The option value is either a full name such as 'America/Los_Angeles', or a custom timezone id such as 'GMT-08:00'.</td>
</tr>
+ <tr>
+ <td><h5>snapshot.expire.clean-empty-directories</h5></td>
+ <td style="word-wrap: break-word;">true</td>
+ <td>Boolean</td>
+ <td>Whether to try to clean empty directories when expiring snapshots. Note that trying to clean directories might throw exceptions in filesystem, but in most cases it won't cause problems.</td>
+ </tr>
<tr>
<td><h5>snapshot.expire.execution-mode</h5></td>
<td style="word-wrap: break-word;">sync</td>
diff --git a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
index 84497806f..550c76c95 100644
--- a/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-common/src/main/java/org/apache/paimon/CoreOptions.java
@@ -213,6 +213,15 @@ public class CoreOptions implements Serializable {
.withDescription(
"The maximum number of snapshots allowed to expire at a time.");
+ public static final ConfigOption<Boolean> SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES =
+ key("snapshot.expire.clean-empty-directories")
+ .booleanType()
+ .defaultValue(true)
+ .withDescription(
+ "Whether to try to clean empty directories when expiring snapshots. "
+ + "Note that trying to clean directories might throw exceptions in filesystem, "
+ + "but in most cases it won't cause problems.");
+
public static final ConfigOption<Duration> CONTINUOUS_DISCOVERY_INTERVAL =
key("continuous.discovery-interval")
.durationType()
@@ -1135,6 +1144,10 @@ public class CoreOptions implements Serializable {
return options.get(SNAPSHOT_EXPIRE_LIMIT);
}
+ public boolean snapshotExpireCleanEmptyDirectories() {
+ return options.get(SNAPSHOT_EXPIRE_CLEAN_EMPTY_DIRECTORIES);
+ }
+
public int manifestMergeMinCount() {
return options.get(MANIFEST_MERGE_MIN_COUNT);
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
index c5a976823..3615b488e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
+++ b/paimon-core/src/main/java/org/apache/paimon/AbstractFileStore.java
@@ -187,7 +187,8 @@ public abstract class AbstractFileStore<T> implements FileStore<T> {
snapshotManager(),
newSnapshotDeletion(),
newTagManager(),
- options.snapshotExpireLimit());
+ options.snapshotExpireLimit(),
+ options.snapshotExpireCleanEmptyDirectories());
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
index c326eb35e..522907587 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreExpireImpl.java
@@ -59,6 +59,7 @@ public class FileStoreExpireImpl implements FileStoreExpire {
private final TagManager tagManager;
private final int expireLimit;
+ private final boolean snapshotExpireCleanEmptyDirectories;
private Lock lock;
@@ -69,7 +70,8 @@ public class FileStoreExpireImpl implements FileStoreExpire {
SnapshotManager snapshotManager,
SnapshotDeletion snapshotDeletion,
TagManager tagManager,
- int expireLimit) {
+ int expireLimit,
+ boolean snapshotExpireCleanEmptyDirectories) {
Preconditions.checkArgument(
numRetainedMin >= 1,
"The minimum number of completed snapshots to retain should be >= 1.");
@@ -88,6 +90,7 @@ public class FileStoreExpireImpl implements FileStoreExpire {
this.snapshotDeletion = snapshotDeletion;
this.tagManager = tagManager;
this.expireLimit = expireLimit;
+ this.snapshotExpireCleanEmptyDirectories = snapshotExpireCleanEmptyDirectories;
}
@Override
@@ -211,7 +214,9 @@ public class FileStoreExpireImpl implements FileStoreExpire {
// data files and changelog files in bucket directories has been deleted
// then delete changed bucket directories if they are empty
- snapshotDeletion.cleanDataDirectories();
+ if (snapshotExpireCleanEmptyDirectories) {
+ snapshotDeletion.cleanDataDirectories();
+ }
// delete manifests and indexFiles
List<Snapshot> skippingSnapshots =
diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
index 6bd62a54b..0a9b2bf5b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
+++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java
@@ -136,6 +136,14 @@ public class TestFileStore extends KeyValueFileStore {
public FileStoreExpireImpl newExpire(
int numRetainedMin, int numRetainedMax, long millisRetained) {
+ return newExpire(numRetainedMin, numRetainedMax, millisRetained, true);
+ }
+
+ public FileStoreExpireImpl newExpire(
+ int numRetainedMin,
+ int numRetainedMax,
+ long millisRetained,
+ boolean snapshotExpireCleanEmptyDirectories) {
return new FileStoreExpireImpl(
numRetainedMin,
numRetainedMax,
@@ -143,7 +151,8 @@ public class TestFileStore extends KeyValueFileStore {
snapshotManager(),
newSnapshotDeletion(),
new TagManager(fileIO, options.path()),
- Integer.MAX_VALUE);
+ Integer.MAX_VALUE,
+ snapshotExpireCleanEmptyDirectories);
}
public List<Snapshot> commitData(
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
index 97708bfc2..7556e6ead 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
@@ -160,6 +160,82 @@ public class FileDeletionTest {
assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
}
+ /**
+ * This test checks FileStoreExpire can ignore clean data directories in snapshot expire
+ * situation. The partition keys are (dt, hr). Test process:
+ *
+ * <ul>
+ * <li>1. Generate snapshot 1 with (0401, 8/12), (0402, 8/12). Each partition has two buckets.
+ * <li>2. Generate snapshot 2 by deleting all data of partition dt=0401 (thus directory
+ * dt=0401 will be exist after expiring).
+ * <li>3. Generate snapshot 3 by deleting all data of partition dt=0402/hr=8 (thus directory
+ * dt=0402/hr=8 will be exist after expiring).
+ * <li>4. Generate snapshot 4 by deleting all data of partition dt=0402/hr=12/bucket-0 (thus
+ * directory dt=0402/hr=12/bucket-0 will be exist after expiring).
+ * <li>5. Expire snapshot 1-3 (dt=0402/hr=20/bucket-1 survives) and set ignore clean data
+ * directories, then check.
+ * </ul>
+ */
+ @Test
+ public void testCleanDirectories() throws Exception {
+ TestFileStore store = createStore(TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED);
+ TestKeyValueGenerator gen =
+ new TestKeyValueGenerator(TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED);
+ FileStorePathFactory pathFactory = store.pathFactory();
+
+ // step 1: generate snapshot 1 by writing 5 randomly generated records to each bucket
+ // writers for each bucket
+ Map<BinaryRow, Map<Integer, RecordWriter<KeyValue>>> writers = new HashMap<>();
+
+ List<BinaryRow> partitions = new ArrayList<>();
+ for (String dt : Arrays.asList("0401", "0402")) {
+ for (int hr : Arrays.asList(8, 12)) {
+ for (int bucket : Arrays.asList(0, 1)) {
+ List<KeyValue> kvs = partitionedData(5, gen, dt, hr);
+ BinaryRow partition = gen.getPartition(kvs.get(0));
+ partitions.add(partition);
+ writeData(store, kvs, partition, bucket, writers);
+ }
+ }
+ }
+
+ commitData(store, commitIdentifier++, writers);
+ // check all paths exist
+ for (BinaryRow partition : partitions) {
+ for (int bucket : Arrays.asList(0, 1)) {
+ assertPathExists(fileIO, pathFactory.bucketPath(partition, bucket));
+ }
+ }
+
+ // step 2: generate snapshot 2 by cleaning partition dt=0401 (through overwriting with an
+ // empty ManifestCommittable)
+ FileStoreCommitImpl commit = store.newCommit();
+ Map<String, String> partitionSpec = new HashMap<>();
+ partitionSpec.put("dt", "0401");
+ commit.overwrite(
+ partitionSpec, new ManifestCommittable(commitIdentifier++), Collections.emptyMap());
+
+ // step 3: generate snapshot 3 by cleaning partition dt=0402/hr=8
+ partitionSpec.put("dt", "0402");
+ partitionSpec.put("hr", "8");
+ commit.overwrite(
+ partitionSpec, new ManifestCommittable(commitIdentifier++), Collections.emptyMap());
+
+ // step 4: generate snapshot 4 by cleaning dt=0402/hr=12/bucket-0
+ BinaryRow partition = partitions.get(7);
+ cleanBucket(store, partition, 0);
+
+ // step 5: expire and check file paths
+ store.newExpire(1, 1, Long.MAX_VALUE, false).expire();
+ // whole dt=0401 is exist
+ assertPathExists(fileIO, new Path(root, "dt=0401"));
+ // whole dt=0402/hr=8 is exist
+ assertPathExists(fileIO, new Path(root, "dt=0402/hr=8"));
+ // for dt=0402/hr=12, bucket-0 is exist and bucket-1 survives
+ assertPathExists(fileIO, pathFactory.bucketPath(partition, 0));
+ assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
+ }
+
// only exists bucket directories
@Test
public void testNoPartitions() throws Exception {