[incubator-doris] branch master updated: [Bug] Fix the bug of rowset file being deleted by mistake (#5541)

[ya...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 3a96b6d  [Bug] Fix the bug of rowset file being deleted by mistake (#5541)
3a96b6d is described below

commit 3a96b6dfbdd0641d9d7f39e5c54ed2e70a46d294
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Tue Mar 23 09:33:36 2021 +0800

    [Bug] Fix the bug of rowset file being deleted by mistake (#5541)
    
    * [Bug] Fix the bug of rowset file being deleted by mistake
---
 be/src/olap/rowset/unique_rowset_id_generator.cpp                 | 8 ++++++++
 be/src/olap/rowset/unique_rowset_id_generator.h                   | 6 ++++++
 .../src/test/java/org/apache/doris/catalog/CreateTableTest.java   | 2 +-
 .../src/test/java/org/apache/doris/planner/PlannerTest.java       | 4 ++--
 4 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/be/src/olap/rowset/unique_rowset_id_generator.cpp b/be/src/olap/rowset/unique_rowset_id_generator.cpp
index c21b8ca..9db7faf 100644
--- a/be/src/olap/rowset/unique_rowset_id_generator.cpp
+++ b/be/src/olap/rowset/unique_rowset_id_generator.cpp
@@ -64,6 +64,14 @@ bool UniqueRowsetIdGenerator::id_in_use(const RowsetId& rowset_id) const {
 }
 
 void UniqueRowsetIdGenerator::release_id(const RowsetId& rowset_id) {
+    // Only release the rowsetid generated after this startup.
+    // So we need to check version/mid/low part first
+    if (rowset_id.version < _version) {
+        return;
+    }
+    if ((rowset_id.mi != _backend_uid.hi) || (rowset_id.lo != _backend_uid.lo)) {
+        return;
+    }
     std::lock_guard<SpinLock> l(_lock);
     _valid_rowset_id_hi.erase(rowset_id.hi);
 }
diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h
index 0793af0..cb5d566 100644
--- a/be/src/olap/rowset/unique_rowset_id_generator.h
+++ b/be/src/olap/rowset/unique_rowset_id_generator.h
@@ -38,7 +38,13 @@ private:
     mutable SpinLock _lock;
     const UniqueId _backend_uid;
     const int64_t _version = 2; // modify it when create new version id generator
+    // A monotonically increasing integer generator,
+    // This integer will be part of a rowset id.
     std::atomic<int64_t> _inc_id;
+    // Save the high part of rowset ids generated since last process startup.
+    // Therefore, we cannot strictly rely on _valid_rowset_id_hi
+    // to determine whether the rowset id is being used.
+    // But to use id_in_use() and release_id() to check it.
     std::unordered_set<int64_t> _valid_rowset_id_hi;
 
     DISALLOW_COPY_AND_ASSIGN(UniqueRowsetIdGenerator);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index 6bfdb87..0dabc18 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -68,7 +68,7 @@ public class CreateTableTest {
     public void testDuplicateCreateTable() throws Exception{
         // test
         Catalog catalog = Catalog.getCurrentCatalog();
-        String sql = "create table if not exists test.tbl1\n" + "(k1 int, k2 int)\n" + "duplicate key(k1)\n"
+        String sql = "create table if not exists test.tbl1_colocate\n" + "(k1 int, k2 int)\n" + "duplicate key(k1)\n"
                 + "distributed by hash(k2) buckets 1\n" + "properties('replication_num' = '1','colocate_with'='test'); ";
         createTable(sql);
         Set<Long> tabletIdSetAfterCreateFirstTable = catalog.getTabletInvertedIndex().getReplicaMetaTable().rowKeySet();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
index eb843fc..135a50b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/PlannerTest.java
@@ -316,7 +316,7 @@ public class PlannerTest {
     }
 
     @Test
-    public void testWithStmtSoltIsAllowNull() throws Exception {
+    public void testWithStmtSlotIsAllowNull() throws Exception {
         // union
         String sql1 = "with a as (select NULL as user_id ), " +
                 "b as ( select '543' as user_id) " +
@@ -326,7 +326,7 @@ public class PlannerTest {
         stmtExecutor1.execute();
         Planner planner1 = stmtExecutor1.planner();
         List<PlanFragment> fragments1 = planner1.getFragments();
-        String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(false, false));
+        String plan1 = planner1.getExplainString(fragments1, new ExplainOptions(true, false));
         Assert.assertEquals(3, StringUtils.countMatches(plan1, "nullIndicatorBit=0"));
     }
 

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org