You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/06/04 08:25:05 UTC

[incubator-iotdb] branch master updated: [IOTDB-672] Fix delete storage group bug during restart (#1310)

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 8f3edd7  [IOTDB-672] Fix delete storage group bug during restart (#1310)
8f3edd7 is described below

commit 8f3edd731d7fbe5f091efa1dfb4ccafd6ac82784
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Jun 4 16:24:54 2020 +0800

    [IOTDB-672] Fix delete storage group bug during restart (#1310)
    
    * fix delete all timeseries in storage group and recover bug
---
 docs/SystemDesign/SchemaManager/SchemaManager.md   |  4 +-
 .../zh/SystemDesign/SchemaManager/SchemaManager.md |  4 +-
 .../org/apache/iotdb/db/metadata/MManager.java     | 48 +++++++++-------------
 .../apache/iotdb/db/qp/executor/PlanExecutor.java  | 11 ++---
 4 files changed, 27 insertions(+), 40 deletions(-)

diff --git a/docs/SystemDesign/SchemaManager/SchemaManager.md b/docs/SystemDesign/SchemaManager/SchemaManager.md
index cec2c86..c8b6616 100644
--- a/docs/SystemDesign/SchemaManager/SchemaManager.md
+++ b/docs/SystemDesign/SchemaManager/SchemaManager.md
@@ -55,11 +55,11 @@ In the process of initializing, MManager will replay the mlog to load the metada
 		* if succeed
 			* delete the LeafMNode
 			* read tlog using offset in the LeafMNode, update tag inverted index
-			* if the storage group becomes empty after deleting, return the name of it
+			* if the storage group becomes empty after deleting, record its name
 		* if failed
 			* return the full path of failed timeseries
-	* iterate the returned empty storage group list, and delete them
 	* if not restart
+	   * delete the recorded empty storage group
 		* persist log into mlog
 		* currently, we won't delete the tag/attribute info of that timeseries in tlog
 	
diff --git a/docs/zh/SystemDesign/SchemaManager/SchemaManager.md b/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
index 8a4cba2..afda6c1 100644
--- a/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
+++ b/docs/zh/SystemDesign/SchemaManager/SchemaManager.md
@@ -54,11 +54,11 @@ IoTDB 的元数据统一由 MManger 管理,包括以下几个部分:
 		* 若获取成功
 			* 删除MTree中对应的LeafMNode
 			* 根据LeafMNode中的offset,读取标签文件,更新 tagIndex 索引
-			* 若发现某一个存储组为空,则返回为空的存储组名
+			* 若发现某一个存储组为空,则记录下该空的存储组名
 		* 若获取失败
 			* 返回此未删除成功的时间序列
-	* 遍历上一步返回的为空的存储组,依次删除
 	* 如果非重启(需要记录日志)
+	      *  删除记录下的空的存储组
 		* 将所删除的时间序列信息记录到 mlog 中
 		* 目前并不会删除 tlog 中关于此时间序列的标签/属性信息。
 	
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 8bfd480..c754c76 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -72,7 +72,7 @@ public class MManager {
   private MTree mtree;
   private MLogWriter logWriter;
   private TagLogFile tagLogFile;
-  private boolean writeToLog;
+  private boolean isRecovering;
   // device -> DeviceMNode
   private RandomDeleteCache<String, MNode> mNodeCache;
 
@@ -108,7 +108,7 @@ public class MManager {
     logFilePath = schemaDir + File.separator + MetadataConstant.METADATA_LOG;
 
     // do not write log when recover
-    writeToLog = false;
+    isRecovering = true;
 
     int cacheSize = config.getmManagerCacheSize();
     mNodeCache =
@@ -156,7 +156,7 @@ public class MManager {
       }
 
       logWriter = new MLogWriter(config.getSchemaDir(), MetadataConstant.METADATA_LOG);
-      writeToLog = true;
+      isRecovering = false;
     } catch (IOException | MetadataException e) {
       mtree = new MTree();
       logger.error("Cannot read MTree from file, using an empty new one", e);
@@ -243,12 +243,9 @@ public class MManager {
         createTimeseries(plan, offset);
         break;
       case MetadataOperationType.DELETE_TIMESERIES:
-        Pair<Set<String>, String> pair = deleteTimeseries(args[1]);
-        for (String deleteStorageGroup : pair.left) {
-          StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
-        }
-        if (!pair.right.isEmpty()) {
-          throw new DeleteFailedException(pair.right);
+        String failedTimeseries = deleteTimeseries(args[1]);
+        if (!failedTimeseries.isEmpty()) {
+          throw new DeleteFailedException(failedTimeseries);
         }
         break;
       case MetadataOperationType.SET_STORAGE_GROUP:
@@ -319,7 +316,7 @@ public class MManager {
       }
 
       // write log
-      if (writeToLog) {
+      if (!isRecovering) {
         // either tags or attributes is not empty
         if ((plan.getTags() != null && !plan.getTags().isEmpty())
             || (plan.getAttributes() != null && !plan.getAttributes().isEmpty())) {
@@ -357,11 +354,9 @@ public class MManager {
    * Delete all timeseries under the given path, may cross different storage group
    *
    * @param prefixPath path to be deleted, could be root or a prefix path or a full path
-   * @return 1. The Set contains StorageGroups that contain no more timeseries after this deletion
-   * files of such StorageGroups should be deleted to reclaim disk space. 2. The String is the
-   * deletion failed Timeseries
+   * @return  The String is the deletion failed Timeseries
    */
-  public Pair<Set<String>, String> deleteTimeseries(String prefixPath) throws MetadataException {
+  public String deleteTimeseries(String prefixPath) throws MetadataException {
     lock.writeLock().lock();
     if (isStorageGroup(prefixPath)) {
 
@@ -378,8 +373,6 @@ public class MManager {
       mNodeCache.clear();
     }
     try {
-      Set<String> emptyStorageGroups = new HashSet<>();
-
       List<String> allTimeseries = mtree.getAllTimeseriesName(prefixPath);
       // Monitor storage group seriesPath is not allowed to be deleted
       allTimeseries.removeIf(p -> p.startsWith(MonitorConstants.STAT_STORAGE_GROUP_PREFIX));
@@ -387,15 +380,18 @@ public class MManager {
       Set<String> failedNames = new HashSet<>();
       for (String p : allTimeseries) {
         try {
-          String emptyStorageGroup = deleteOneTimeseriesAndUpdateStatisticsAndLog(p);
-          if (emptyStorageGroup != null) {
-            emptyStorageGroups.add(emptyStorageGroup);
+          String emptyStorageGroup = deleteOneTimeseriesAndUpdateStatistics(p);
+          if (!isRecovering) {
+            if (emptyStorageGroup != null) {
+              StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(emptyStorageGroup);
+            }
+            logWriter.deleteTimeseries(p);
           }
         } catch (DeleteFailedException e) {
           failedNames.add(e.getName());
         }
       }
-      return new Pair<>(emptyStorageGroups, String.join(",", failedNames));
+      return String.join(",", failedNames);
     } catch (IOException e) {
       throw new MetadataException(e.getMessage());
     } finally {
@@ -432,7 +428,7 @@ public class MManager {
    * @param path full path from root to leaf node
    * @return after delete if the storage group is empty, return its name, otherwise return null
    */
-  private String deleteOneTimeseriesAndUpdateStatisticsAndLog(String path)
+  private String deleteOneTimeseriesAndUpdateStatistics(String path)
       throws MetadataException, IOException {
     lock.writeLock().lock();
     try {
@@ -457,10 +453,6 @@ public class MManager {
               .ifPresent(val -> maxSeriesNumberAmongStorageGroup = val);
         }
       }
-
-      if (writeToLog) {
-        logWriter.deleteTimeseries(path);
-      }
       return storageGroupName;
     } finally {
       lock.writeLock().unlock();
@@ -482,7 +474,7 @@ public class MManager {
         ActiveTimeSeriesCounter.getInstance().init(storageGroup);
         seriesNumberInStorageGroups.put(storageGroup, 0);
       }
-      if (writeToLog) {
+      if (!isRecovering) {
         logWriter.setStorageGroup(storageGroup);
       }
     } catch (IOException e) {
@@ -523,7 +515,7 @@ public class MManager {
           }
         }
         // if success
-        if (writeToLog) {
+        if (!isRecovering) {
           logWriter.deleteStorageGroup(storageGroup);
         }
       }
@@ -978,7 +970,7 @@ public class MManager {
     lock.writeLock().lock();
     try {
       getStorageGroupNode(storageGroup).setDataTTL(dataTTL);
-      if (writeToLog) {
+      if (!isRecovering) {
         logWriter.setTTL(storageGroup, dataTTL);
       }
     } finally {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 5d6a268..48697ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -1143,18 +1143,13 @@ public class PlanExecutor implements IPlanExecutor {
     List<Path> deletePathList = deleteTimeSeriesPlan.getPaths();
     try {
       deleteDataOfTimeSeries(deletePathList);
-      Set<String> emptyStorageGroups = new HashSet<>();
       List<String> failedNames = new LinkedList<>();
       for (Path path : deletePathList) {
-        Pair<Set<String>, String> pair = mManager.deleteTimeseries(path.toString());
-        emptyStorageGroups.addAll(pair.left);
-        if (!pair.right.isEmpty()) {
-          failedNames.add(pair.right);
+        String failedTimeseries = mManager.deleteTimeseries(path.toString());
+        if (!failedTimeseries.isEmpty()) {
+          failedNames.add(failedTimeseries);
         }
       }
-      for (String deleteStorageGroup : emptyStorageGroups) {
-        StorageEngine.getInstance().deleteAllDataFilesInOneStorageGroup(deleteStorageGroup);
-      }
       if (!failedNames.isEmpty()) {
         throw new DeleteFailedException(String.join(",", failedNames));
       }