You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/02/17 14:19:29 UTC

[iotdb] branch master updated: [IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)

This is an automated email from the ASF dual-hosted git repository.

haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new 568fd01  [IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)
568fd01 is described below

commit 568fd01c6be5558c56f4b34f8aebd8a684a17b89
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Feb 17 22:18:47 2022 +0800

    [IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)
---
 .../iotdb/db/integration/IoTDBCheckConfigIT.java   |   4 +-
 .../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 283 +++------------------
 2 files changed, 34 insertions(+), 253 deletions(-)

diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
index 934c7b0..c464f6f 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
@@ -70,13 +70,13 @@ public class IoTDBCheckConfigIT {
     console = System.out;
     System.setOut(new PrintStream(bytes));
 
-    systemProperties.put("partition_interval", "604800");
+    systemProperties.put("partition_interval", "9223372036854775807");
     systemProperties.put("timestamp_precision", "ms");
     systemProperties.put("tsfile_storage_fs", "LOCAL");
     systemProperties.put("enable_partition", "false");
     systemProperties.put("max_degree_of_index_node", "256");
     systemProperties.put("tag_attribute_total_size", "700");
-    systemProperties.put("iotdb_version", "0.11.2");
+    systemProperties.put("iotdb_version", "0.13.0");
     systemProperties.put("virtual_storage_group_num", "1");
   }
 
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index 6c88952..d4492c7 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -18,18 +18,12 @@
  */
 package org.apache.iotdb.db.conf;
 
-import org.apache.iotdb.db.conf.directories.DirectoryManager;
 import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
 import org.apache.iotdb.db.exception.ConfigurationException;
-import org.apache.iotdb.db.metadata.logfile.MLogUpgrader;
 import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
 import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
 import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
 import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
-import org.apache.iotdb.tsfile.utils.FilePathUtils;
 
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
@@ -41,14 +35,10 @@ import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.nio.file.Files;
-import java.util.ArrayList;
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Properties;
-import java.util.Set;
 
 public class IoTDBConfigCheck {
 
@@ -62,7 +52,6 @@ public class IoTDBConfigCheck {
   // If user delete folder "data", system.properties can reset.
   private static final String PROPERTIES_FILE_NAME = "system.properties";
   private static final String SCHEMA_DIR = config.getSchemaDir();
-  private static final String SYSTEM_DIR = config.getSystemDir();
   private static final String WAL_DIR = config.getWalDir();
 
   private File propertiesFile;
@@ -213,33 +202,44 @@ public class IoTDBConfigCheck {
             new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) {
       properties.load(inputStreamReader);
     }
-    // check whether upgrading from <=v0.9 to v0.12
+    // check whether upgrading from <=v0.9
     if (!properties.containsKey(IOTDB_VERSION_STRING)) {
       logger.error(
           "DO NOT UPGRADE IoTDB from v0.9 or lower version to v0.12!"
               + " Please upgrade to v0.10 first");
       System.exit(-1);
     }
-    // check whether upgrading from old versions
+    // check whether upgrading from [v0.10, v.12]
     String versionString = properties.getProperty(IOTDB_VERSION_STRING);
-    if (versionString.startsWith("0.12")) {
+    if (versionString.startsWith("0.10") || versionString.startsWith("0.11")) {
+      logger.error("IoTDB version is too old, please upgrade to 0.12 firstly.");
+      System.exit(-1);
+    } else if (versionString.startsWith("0.12")) {
       checkWALNotExists();
-    } else if (versionString.startsWith("0.10") || versionString.startsWith("0.11")) {
-      logger.info(
-          "Upgrading IoTDB from {} to {}, checking files...", versionString, IoTDBConstant.VERSION);
-      checkUnClosedTsFileV2();
-      moveTsFileV2();
-      moveVersionFile();
-      logger.info("checking files successful");
-      logger.info("Start upgrading files...");
-      MLogUpgrader.upgradeMLog();
-      logger.info("Mlog upgraded!");
       upgradePropertiesFile();
     }
     checkProperties();
   }
 
-  /** upgrade 0.10 or 0.11 properties to 0.12 properties */
+  private void checkWALNotExists() {
+    if (SystemFileFactory.INSTANCE.getFile(WAL_DIR).isDirectory()) {
+      File[] sgWALs = SystemFileFactory.INSTANCE.getFile(WAL_DIR).listFiles();
+      if (sgWALs != null) {
+        for (File sgWAL : sgWALs) {
+          // make sure wal directory of each sg is empty
+          if (sgWAL.isDirectory() && sgWAL.list().length != 0) {
+            logger.error(
+                "WAL detected, please stop insertion and run 'SET SYSTEM TO READONLY', then run 'flush' on IoTDB {} before upgrading to {}.",
+                properties.getProperty(IOTDB_VERSION_STRING),
+                IoTDBConstant.VERSION);
+            System.exit(-1);
+          }
+        }
+      }
+    }
+  }
+
+  /** upgrade 0.12 properties to 0.13 properties */
   private void upgradePropertiesFile() throws IOException {
     // create an empty tmpPropertiesFile
     if (tmpPropertiesFile.createNewFile()) {
@@ -249,22 +249,13 @@ public class IoTDBConfigCheck {
       System.exit(-1);
     }
 
-    // virtual storage group num can only set to 1 when upgrading from old version
-    if (!"1".equals(virtualStorageGroupNum)) {
-      logger.error(
-          "virtual storage group num cannot set to {} when upgrading from old version, "
-              + "please set to 1 and restart",
-          virtualStorageGroupNum);
-      System.exit(-1);
-    }
     try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
-      properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
-      properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
-      properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION);
-      properties.setProperty(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
-      properties.setProperty(TAG_ATTRIBUTE_SIZE_STRING, tagAttributeTotalSize);
-      properties.setProperty(TAG_ATTRIBUTE_FLUSH_INTERVAL, tagAttributeFlushInterval);
-      properties.setProperty(MAX_DEGREE_OF_INDEX_STRING, maxDegreeOfIndexNode);
+      systemProperties.forEach(
+          (k, v) -> {
+            if (!properties.containsKey(k)) {
+              properties.setProperty(k, v);
+            }
+          });
       properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
 
       // upgrade finished, delete old system.properties file
@@ -276,7 +267,7 @@ public class IoTDBConfigCheck {
     FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
   }
 
-  /** repair 0.10 properties */
+  /** repair broken properties */
   private void upgradePropertiesFileFromBrokenFile() throws IOException {
     // create an empty tmpPropertiesFile
     if (tmpPropertiesFile.createNewFile()) {
@@ -354,214 +345,4 @@ public class IoTDBConfigCheck {
     throw new ConfigurationException(
         parameter, String.valueOf(badValue), properties.getProperty(parameter));
   }
-
-  /** ensure all TsFiles are closed when starting 0.12 */
-  private void checkUnClosedTsFileV2() {
-    checkWALNotExists();
-    checkUnClosedTsFileV2InFolders(DirectoryManager.getInstance().getAllSequenceFileFolders());
-    checkUnClosedTsFileV2InFolders(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
-  }
-
-  private void checkWALNotExists() {
-    if (SystemFileFactory.INSTANCE.getFile(WAL_DIR).isDirectory()
-        && SystemFileFactory.INSTANCE.getFile(WAL_DIR).list().length != 0) {
-      logger.error(
-          "WAL detected, please stop insertion and run 'SET SYSTEM TO READONLY', then run 'flush' on IoTDB {} before upgrading to {}.",
-          properties.getProperty(IOTDB_VERSION_STRING),
-          IoTDBConstant.VERSION);
-      System.exit(-1);
-    }
-  }
-
-  private void checkUnClosedTsFileV2InFolders(List<String> folders) {
-    for (String baseDir : folders) {
-      File fileFolder = fsFactory.getFile(baseDir);
-      if (!fileFolder.isDirectory()) {
-        continue;
-      }
-      for (File storageGroup : fileFolder.listFiles()) {
-        if (!storageGroup.isDirectory()) {
-          continue;
-        }
-        for (File partitionDir : storageGroup.listFiles()) {
-          if (!partitionDir.isDirectory()) {
-            continue;
-          }
-          File[] tsfiles =
-              fsFactory.listFilesBySuffix(partitionDir.toString(), TsFileConstant.TSFILE_SUFFIX);
-          File[] resources =
-              fsFactory.listFilesBySuffix(partitionDir.toString(), TsFileResource.RESOURCE_SUFFIX);
-          if (tsfiles.length != resources.length) {
-            File[] zeroLevelTsFiles =
-                fsFactory.listFilesBySuffix(
-                    partitionDir.toString(), "0" + TsFileConstant.TSFILE_SUFFIX);
-            File[] zeroLevelResources =
-                fsFactory.listFilesBySuffix(
-                    partitionDir.toString(),
-                    "0" + TsFileConstant.TSFILE_SUFFIX + TsFileResource.RESOURCE_SUFFIX);
-            if (zeroLevelTsFiles.length != zeroLevelResources.length) {
-              logger.error(
-                  "Unclosed Version-2 TsFile detected, please stop insertion, then run 'flush' "
-                      + "on IoTDB {} before upgrading to {}",
-                  properties.getProperty(IOTDB_VERSION_STRING),
-                  IoTDBConstant.VERSION);
-              System.exit(-1);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  /**
-   * If upgrading from v0.11.2 to v0.12, there may be some unsealed merging files. We have to delete
-   * these files before upgrading.
-   */
-  private File[] deleteMergingTsFiles(File[] tsfiles, File[] resources) {
-    Set<String> resourcesSet = new HashSet<>();
-    for (File resource : resources) {
-      resourcesSet.add(resource.getName());
-    }
-    List<File> tsfileList = new ArrayList<>();
-    for (File tsfile : tsfiles) {
-      if (!resourcesSet.contains(tsfile.getName() + TsFileResource.RESOURCE_SUFFIX)) {
-        try {
-          logger.info("Delete merging TsFile {}", tsfile);
-          Files.delete(tsfile.toPath());
-        } catch (Exception e) {
-          logger.error("Failed to delete merging tsfile {} ", tsfile, e);
-          System.exit(-1);
-        }
-      } else {
-        tsfileList.add(tsfile);
-      }
-    }
-    return tsfileList.toArray(new File[tsfileList.size()]);
-  }
-
-  private void moveTsFileV2() {
-    moveFileToUpgradeFolder(DirectoryManager.getInstance().getAllSequenceFileFolders());
-    moveFileToUpgradeFolder(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
-    logger.info("Move version-2 TsFile successfully");
-  }
-
-  @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
-  private void moveFileToUpgradeFolder(List<String> folders) {
-    for (String baseDir : folders) {
-      File fileFolder = fsFactory.getFile(baseDir);
-      if (!fileFolder.isDirectory()) {
-        continue;
-      }
-      for (File storageGroup : fileFolder.listFiles()) {
-        if (!storageGroup.isDirectory()) {
-          continue;
-        }
-        // create virtual storage group folder 0
-        File virtualStorageGroupDir = fsFactory.getFile(storageGroup, "0");
-        if (virtualStorageGroupDir.mkdirs()) {
-          logger.info(
-              "virtual storage directory {} doesn't exist, create it",
-              virtualStorageGroupDir.getPath());
-        } else if (!virtualStorageGroupDir.exists()) {
-          logger.error(
-              "Create virtual storage directory {} failed", virtualStorageGroupDir.getPath());
-        }
-        for (File partitionDir : storageGroup.listFiles()) {
-          if (!partitionDir.isDirectory()) {
-            continue;
-          }
-          File[] oldTsfileArray =
-              fsFactory.listFilesBySuffix(
-                  partitionDir.getAbsolutePath(), TsFileConstant.TSFILE_SUFFIX);
-          File[] oldResourceFileArray =
-              fsFactory.listFilesBySuffix(
-                  partitionDir.getAbsolutePath(), TsFileResource.RESOURCE_SUFFIX);
-          File[] oldModificationFileArray =
-              fsFactory.listFilesBySuffix(
-                  partitionDir.getAbsolutePath(), ModificationFile.FILE_SUFFIX);
-          oldTsfileArray = deleteMergingTsFiles(oldTsfileArray, oldResourceFileArray);
-          // move the old files to upgrade folder if exists
-          if (oldTsfileArray.length + oldResourceFileArray.length + oldModificationFileArray.length
-              != 0) {
-            // create upgrade directory if not exist
-            File upgradeFolder =
-                fsFactory.getFile(virtualStorageGroupDir, IoTDBConstant.UPGRADE_FOLDER_NAME);
-            if (upgradeFolder.mkdirs()) {
-              logger.info("Upgrade Directory {} doesn't exist, create it", upgradeFolder.getPath());
-            } else if (!upgradeFolder.exists()) {
-              logger.error("Create upgrade Directory {} failed", upgradeFolder.getPath());
-            }
-            // move .tsfile to upgrade folder
-            for (File file : oldTsfileArray) {
-              if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
-                logger.error("Failed to move tsfile {} to upgrade folder", file);
-              }
-            }
-            // move .resource to upgrade folder
-            for (File file : oldResourceFileArray) {
-              if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
-                logger.error("Failed to move resource {} to upgrade folder", file);
-              }
-            }
-            // move .mods to upgrade folder
-            for (File file : oldModificationFileArray) {
-              if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
-                logger.error("Failed to move mod file {} to upgrade folder", file);
-              }
-            }
-          }
-          if (partitionDir.listFiles().length == 0) {
-            try {
-              Files.delete(partitionDir.toPath());
-            } catch (IOException e) {
-              logger.error("Delete {} failed", partitionDir);
-            }
-          }
-        }
-      }
-    }
-  }
-
-  @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
-  private void moveVersionFile() {
-    File sgDir =
-        SystemFileFactory.INSTANCE.getFile(
-            FilePathUtils.regularizePath(SYSTEM_DIR) + "storage_groups");
-    if (sgDir.isDirectory()) {
-      for (File sg : sgDir.listFiles()) {
-        if (!sg.isDirectory()) {
-          continue;
-        }
-        for (File partition : sg.listFiles()) {
-          if (!partition.isDirectory()) {
-            continue;
-          }
-          File virtualSg = SystemFileFactory.INSTANCE.getFile(sg, "0");
-          if (!virtualSg.exists()) {
-            virtualSg.mkdir();
-          }
-          File newPartition = SystemFileFactory.INSTANCE.getFile(virtualSg, partition.getName());
-          if (!newPartition.exists()) {
-            newPartition.mkdir();
-          }
-          for (File versionFile : partition.listFiles()) {
-            if (versionFile.isDirectory()) {
-              continue;
-            }
-            if (!versionFile.renameTo(
-                SystemFileFactory.INSTANCE.getFile(newPartition, versionFile.getName()))) {
-              logger.error("Rename {} failed", versionFile);
-            }
-          }
-          if (partition.listFiles().length == 0) {
-            try {
-              Files.delete(partition.toPath());
-            } catch (IOException e) {
-              logger.error("Delete {} failed", partition);
-            }
-          }
-        }
-      }
-    }
-  }
 }