You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/07/08 02:41:05 UTC

(paimon) branch master updated: [core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)

This is an automated email from the ASF dual-hosted git repository.

lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new e7e49e643 [core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)
e7e49e643 is described below

commit e7e49e643adda29e4347d1eb982eeb2351d49209
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Mon Jul 8 10:40:59 2024 +0800

    [core] Fix that FileDeletionBase#dataFileSkipper might skip tag when deleting tag (#3650)
---
 .../apache/paimon/operation/FileDeletionBase.java  | 19 ++++---
 .../apache/paimon/operation/FileDeletionTest.java  | 62 ++++++++++++++++++++++
 2 files changed, 73 insertions(+), 8 deletions(-)

diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
index 9d923f46d..f8d777ab7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileDeletionBase.java
@@ -75,8 +75,8 @@ public abstract class FileDeletionBase<T extends Snapshot> {
     protected final Executor ioExecutor;
     protected boolean changelogDecoupled;
 
-    /** Used to record which tag is cached in tagged snapshots list. */
-    private int cachedTagIndex = -1;
+    /** Used to record which tag is cached. */
+    private long cachedTag = 0;
 
     /** Used to cache data files used by current tag. */
     private final Map<BinaryRow, Map<Integer, Set<String>>> cachedTagDataFiles = new HashMap<>();
@@ -326,13 +326,16 @@ public abstract class FileDeletionBase<T extends Snapshot> {
             List<Snapshot> taggedSnapshots, long expiringSnapshotId) throws Exception {
         int index = TagManager.findPreviousTag(taggedSnapshots, expiringSnapshotId);
         // refresh tag data files
-        if (index >= 0 && cachedTagIndex != index) {
-            cachedTagIndex = index;
-            cachedTagDataFiles.clear();
-            addMergedDataFiles(cachedTagDataFiles, taggedSnapshots.get(index));
+        if (index >= 0) {
+            Snapshot previousTag = taggedSnapshots.get(index);
+            if (previousTag.id() != cachedTag) {
+                cachedTag = previousTag.id();
+                cachedTagDataFiles.clear();
+                addMergedDataFiles(cachedTagDataFiles, previousTag);
+            }
+            return entry -> containsDataFile(cachedTagDataFiles, entry);
         }
-
-        return entry -> index >= 0 && containsDataFile(cachedTagDataFiles, entry);
+        return entry -> false;
     }
 
     /**
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
index a9be8d3ac..a648da19b 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java
@@ -34,9 +34,12 @@ import org.apache.paimon.manifest.ManifestFile;
 import org.apache.paimon.manifest.ManifestFileMeta;
 import org.apache.paimon.manifest.ManifestList;
 import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction;
+import org.apache.paimon.options.ExpireConfig;
 import org.apache.paimon.schema.Schema;
 import org.apache.paimon.schema.SchemaManager;
 import org.apache.paimon.schema.TableSchema;
+import org.apache.paimon.table.ExpireSnapshots;
+import org.apache.paimon.table.ExpireSnapshotsImpl;
 import org.apache.paimon.types.RowType;
 import org.apache.paimon.utils.FileStorePathFactory;
 import org.apache.paimon.utils.RecordWriter;
@@ -636,6 +639,65 @@ public class FileDeletionTest {
         assertPathExists(fileIO, dataFileA);
     }
 
+    @Test
+    public void testExpireWithDeletingTags() throws Exception {
+        TestFileStore store = createStore(TestKeyValueGenerator.GeneratorMode.NON_PARTITIONED, 2);
+        tagManager = new TagManager(fileIO, store.options().path());
+        SnapshotManager snapshotManager = store.snapshotManager();
+        TestKeyValueGenerator gen =
+                new TestKeyValueGenerator(TestKeyValueGenerator.GeneratorMode.NON_PARTITIONED);
+        BinaryRow partition = gen.getPartition(gen.next());
+
+        // step 1: commit A to bucket 0 and create tag1
+        Map<BinaryRow, Map<Integer, RecordWriter<KeyValue>>> writers = new HashMap<>();
+        List<KeyValue> kvs = partitionedData(5, gen);
+        writeData(store, kvs, partition, 0, writers);
+        commitData(store, commitIdentifier++, writers);
+        createTag(snapshotManager.snapshot(1), "tag1", store.options().tagDefaultTimeRetained());
+
+        // step 2: commit B to bucket 1 and create tag2
+        writers.clear();
+        kvs = partitionedData(5, gen);
+        writeData(store, kvs, partition, 1, writers);
+        commitData(store, commitIdentifier++, writers);
+        createTag(snapshotManager.snapshot(2), "tag2", store.options().tagDefaultTimeRetained());
+
+        // step 3: commit -B
+        cleanBucket(store, partition, 1);
+
+        // step 4: commit -A
+        cleanBucket(store, partition, 0);
+
+        // action: expire snapshot 1 -> delete tag1 -> expire snapshot 2
+        // result: exist A & B (because of tag2)
+        ExpireSnapshots expireSnapshots =
+                new ExpireSnapshotsImpl(
+                        snapshotManager, store.newSnapshotDeletion(), tagManager, true);
+        expireSnapshots
+                .config(
+                        ExpireConfig.builder()
+                                .snapshotRetainMax(3)
+                                .snapshotRetainMin(3)
+                                .snapshotTimeRetain(Duration.ofMillis(Long.MAX_VALUE))
+                                .build())
+                .expire();
+        tagManager.deleteTag(
+                "tag1", store.newTagDeletion(), snapshotManager, Collections.emptyList());
+        expireSnapshots
+                .config(
+                        ExpireConfig.builder()
+                                .snapshotRetainMax(2)
+                                .snapshotRetainMin(2)
+                                .snapshotTimeRetain(Duration.ofMillis(Long.MAX_VALUE))
+                                .build())
+                .expire();
+
+        assertThat(snapshotManager.snapshotCount()).isEqualTo(2);
+        FileStorePathFactory pathFactory = store.pathFactory();
+        assertPathExists(fileIO, pathFactory.bucketPath(partition, 0));
+        assertPathExists(fileIO, pathFactory.bucketPath(partition, 1));
+    }
+
     private TestFileStore createStore(TestKeyValueGenerator.GeneratorMode mode) throws Exception {
         return createStore(mode, 2);
     }