You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/07/08 02:41:05 UTC
(paimon) branch master updated: [core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new e7e49e643 [core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)
e7e49e643 is described below
commit e7e49e643adda29e4347d1eb982eeb2351d49209
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Mon Jul 8 10:40:59 2024 +0800
[core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)
---
.../apache/paimon/operation/FileDeletionBase.java | 19 ++++---
.../apache/paimon/operation/FileDeletionTest.java | 62 ++++++++++++++++++++++
2 files changed, 73 insertions(+), 8 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index 9d923f46d..f8d777ab7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -75,8 +75,8 @@ public abstract class FileDeletionBase<T extends Snapshot> {
protected final Executor ioExecutor;
protected boolean changelogDecoupled;
- /** Used to record which tag is cached in tagged snapshots list. */
- private int cachedTagIndex = -1;
+ /** Used to record which tag is cached. */
+ private long cachedTag = 0;
/** Used to cache data files used by current tag. */
private final Map<BinaryRow, Map<Integer, Set<String>>> cachedTagDataFiles = new HashMap<>();
@@ -326,13 +326,16 @@ public abstract class FileDeletionBase<T extends Snapshot> {
List<Snapshot> taggedSnapshots, long expiringSnapshotId) throws Exception {
int index = TagManager.findPreviousTag(taggedSnapshots, expiringSnapshotId);
// refresh tag data files
- if (index >= 0 && cachedTagIndex != index) {
- cachedTagIndex = index;
- cachedTagDataFiles.clear();
- addMergedDataFiles(cachedTagDataFiles, taggedSnapshots.get(index));
+ if (index >= 0) {
+ Snapshot previousTag = taggedSnapshots.get(index);
+ if (previousTag.id() != cachedTag) {
+ cachedTag = previousTag.id();
+ cachedTagDataFiles.clear();
+ addMergedDataFiles(cachedTagDataFiles, previousTag);
+ }
+ return entry -> containsDataFile(cachedTagDataFiles, entry);
}
-
- return entry -> index >= 0 && containsDataFile(cachedTagDataFiles, entry);
+ return entry -> false;
}
/**
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
index a9be8d3ac..a648da19b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
@@ -34,9 +34,12 @@ import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestList;
import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction;
+import org.apache.paimon.options.ExpireConfig;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.ExpireSnapshots;
+import org.apache.paimon.table.ExpireSnapshotsImpl;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FileStorePathFactory;
import org.apache.paimon.utils.RecordWriter;
@@ -636,6 +639,65 @@ public class FileDeletionTest {
assertPathExists(fileIO, dataFileA);
}
+ @Test
+ public void testExpireWithDeletingTags() throws Exception {
+ TestFileStore store = createStore(TestKeyValueGenerator.GeneratorMode.NON_PARTITIONED, 2);
+ tagManager = new TagManager(fileIO, store.options().path());
+ SnapshotManager snapshotManager = store.snapshotManager();
+ TestKeyValueGenerator gen =
+ new TestKeyValueGenerator(TestKeyValueGenerator.GeneratorMode.NON_PARTITIONED);
+ BinaryRow partition = gen.getPartition(gen.next());
+
+ // step 1: commit A to bucket 0 and create tag1
+ Map<BinaryRow, Map<Integer, RecordWriter<KeyValue>>> writers = new HashMap<>();
+ List<KeyValue> kvs = partitionedData(5, gen);
+ writeData(store, kvs, partition, 0, writers);
+ commitData(store, commitIdentifier++, writers);
+ createTag(snapshotManager.snapshot(1), "tag1", store.options().tagDefaultTimeRetained());
+
+ // step 2: commit B to bucket 1 and create tag2
+ writers.clear();
+ kvs = partitionedData(5, gen);
+ writeData(store, kvs, partition, 1, writers);
+ commitData(store, commitIdentifier++, writers);
+ createTag(snapshotManager.snapshot(2), "tag2", store.options().tagDefaultTimeRetained());
+
+ // step 3: commit -B
+ cleanBucket(store, partition, 1);
+
+ // step 4: commit -A
+ cleanBucket(store, partition, 0);
+
+ // action: expire snapshot 1 -> delete tag1 -> expire snapshot 2
+ // result: exist A & B (because of tag2)
+ ExpireSnapshots expireSnapshots =
+ new ExpireSnapshotsImpl(
+ snapshotManager, store.newSnapshotDeletion(), tagManager, true);
+ expireSnapshots
+ .config(
+ ExpireConfig.builder()
+ .snapshotRetainMax(3)
+ .snapshotRetainMin(3)
+ .snapshotTimeRetain(Duration.ofMillis(Long.MAX_VALUE))
+ .build())
+ .expire();
+ tagManager.deleteTag(
+ "tag1", store.newTagDeletion(), snapshotManager, Collections.emptyList());
+ expireSnapshots
+ .config(
+ ExpireConfig.builder()
+ .snapshotRetainMax(2)
+ .snapshotRetainMin(2)
+ .snapshotTimeRetain(Duration.ofMillis(Long.MAX_VALUE))
+ .build())
+ .expire();
+
+ assertThat(snapshotManager.snapshotCount()).isEqualTo(2);
+ FileStorePathFactory pathFactory = store.pathFactory();
+ assertPathExists(fileIO, pathFactory.bucketPath(partition, 0));
+ assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
+ }
+
private TestFileStore createStore(TestKeyValueGenerator.GeneratorMode mode) throws Exception {
return createStore(mode, 2);
}