You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/02/17 14:19:29 UTC
[iotdb] branch master updated: [IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 568fd01 [IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)
568fd01 is described below
commit 568fd01c6be5558c56f4b34f8aebd8a684a17b89
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Feb 17 22:18:47 2022 +0800
[IOTDB-2565] Only support upgrading 0.13 from 0.12 (#5077)
---
.../iotdb/db/integration/IoTDBCheckConfigIT.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 283 +++------------------
2 files changed, 34 insertions(+), 253 deletions(-)
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
index 934c7b0..c464f6f 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBCheckConfigIT.java
@@ -70,13 +70,13 @@ public class IoTDBCheckConfigIT {
console = System.out;
System.setOut(new PrintStream(bytes));
- systemProperties.put("partition_interval", "604800");
+ systemProperties.put("partition_interval", "9223372036854775807");
systemProperties.put("timestamp_precision", "ms");
systemProperties.put("tsfile_storage_fs", "LOCAL");
systemProperties.put("enable_partition", "false");
systemProperties.put("max_degree_of_index_node", "256");
systemProperties.put("tag_attribute_total_size", "700");
- systemProperties.put("iotdb_version", "0.11.2");
+ systemProperties.put("iotdb_version", "0.13.0");
systemProperties.put("virtual_storage_group_num", "1");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
index 6c88952..d4492c7 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -18,18 +18,12 @@
*/
package org.apache.iotdb.db.conf;
-import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
-import org.apache.iotdb.db.engine.modification.ModificationFile;
-import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.ConfigurationException;
-import org.apache.iotdb.db.metadata.logfile.MLogUpgrader;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.apache.iotdb.tsfile.fileSystem.fsFactory.FSFactory;
-import org.apache.iotdb.tsfile.utils.FilePathUtils;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
@@ -41,14 +35,10 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.file.Files;
-import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
-import java.util.Set;
public class IoTDBConfigCheck {
@@ -62,7 +52,6 @@ public class IoTDBConfigCheck {
// If user delete folder "data", system.properties can reset.
private static final String PROPERTIES_FILE_NAME = "system.properties";
private static final String SCHEMA_DIR = config.getSchemaDir();
- private static final String SYSTEM_DIR = config.getSystemDir();
private static final String WAL_DIR = config.getWalDir();
private File propertiesFile;
@@ -213,33 +202,44 @@ public class IoTDBConfigCheck {
new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) {
properties.load(inputStreamReader);
}
- // check whether upgrading from <=v0.9 to v0.12
+ // check whether upgrading from <=v0.9
if (!properties.containsKey(IOTDB_VERSION_STRING)) {
logger.error(
"DO NOT UPGRADE IoTDB from v0.9 or lower version to v0.12!"
+ " Please upgrade to v0.10 first");
System.exit(-1);
}
- // check whether upgrading from old versions
+ // check whether upgrading from [v0.10, v.12]
String versionString = properties.getProperty(IOTDB_VERSION_STRING);
- if (versionString.startsWith("0.12")) {
+ if (versionString.startsWith("0.10") || versionString.startsWith("0.11")) {
+ logger.error("IoTDB version is too old, please upgrade to 0.12 firstly.");
+ System.exit(-1);
+ } else if (versionString.startsWith("0.12")) {
checkWALNotExists();
- } else if (versionString.startsWith("0.10") || versionString.startsWith("0.11")) {
- logger.info(
- "Upgrading IoTDB from {} to {}, checking files...", versionString, IoTDBConstant.VERSION);
- checkUnClosedTsFileV2();
- moveTsFileV2();
- moveVersionFile();
- logger.info("checking files successful");
- logger.info("Start upgrading files...");
- MLogUpgrader.upgradeMLog();
- logger.info("Mlog upgraded!");
upgradePropertiesFile();
}
checkProperties();
}
- /** upgrade 0.10 or 0.11 properties to 0.12 properties */
+ private void checkWALNotExists() {
+ if (SystemFileFactory.INSTANCE.getFile(WAL_DIR).isDirectory()) {
+ File[] sgWALs = SystemFileFactory.INSTANCE.getFile(WAL_DIR).listFiles();
+ if (sgWALs != null) {
+ for (File sgWAL : sgWALs) {
+ // make sure wal directory of each sg is empty
+ if (sgWAL.isDirectory() && sgWAL.list().length != 0) {
+ logger.error(
+ "WAL detected, please stop insertion and run 'SET SYSTEM TO READONLY', then run 'flush' on IoTDB {} before upgrading to {}.",
+ properties.getProperty(IOTDB_VERSION_STRING),
+ IoTDBConstant.VERSION);
+ System.exit(-1);
+ }
+ }
+ }
+ }
+ }
+
+ /** upgrade 0.12 properties to 0.13 properties */
private void upgradePropertiesFile() throws IOException {
// create an empty tmpPropertiesFile
if (tmpPropertiesFile.createNewFile()) {
@@ -249,22 +249,13 @@ public class IoTDBConfigCheck {
System.exit(-1);
}
- // virtual storage group num can only set to 1 when upgrading from old version
- if (!"1".equals(virtualStorageGroupNum)) {
- logger.error(
- "virtual storage group num cannot set to {} when upgrading from old version, "
- + "please set to 1 and restart",
- virtualStorageGroupNum);
- System.exit(-1);
- }
try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
- properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
- properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
- properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION);
- properties.setProperty(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
- properties.setProperty(TAG_ATTRIBUTE_SIZE_STRING, tagAttributeTotalSize);
- properties.setProperty(TAG_ATTRIBUTE_FLUSH_INTERVAL, tagAttributeFlushInterval);
- properties.setProperty(MAX_DEGREE_OF_INDEX_STRING, maxDegreeOfIndexNode);
+ systemProperties.forEach(
+ (k, v) -> {
+ if (!properties.containsKey(k)) {
+ properties.setProperty(k, v);
+ }
+ });
properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
// upgrade finished, delete old system.properties file
@@ -276,7 +267,7 @@ public class IoTDBConfigCheck {
FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
- /** repair 0.10 properties */
+ /** repair broken properties */
private void upgradePropertiesFileFromBrokenFile() throws IOException {
// create an empty tmpPropertiesFile
if (tmpPropertiesFile.createNewFile()) {
@@ -354,214 +345,4 @@ public class IoTDBConfigCheck {
throw new ConfigurationException(
parameter, String.valueOf(badValue), properties.getProperty(parameter));
}
-
- /** ensure all TsFiles are closed when starting 0.12 */
- private void checkUnClosedTsFileV2() {
- checkWALNotExists();
- checkUnClosedTsFileV2InFolders(DirectoryManager.getInstance().getAllSequenceFileFolders());
- checkUnClosedTsFileV2InFolders(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
- }
-
- private void checkWALNotExists() {
- if (SystemFileFactory.INSTANCE.getFile(WAL_DIR).isDirectory()
- && SystemFileFactory.INSTANCE.getFile(WAL_DIR).list().length != 0) {
- logger.error(
- "WAL detected, please stop insertion and run 'SET SYSTEM TO READONLY', then run 'flush' on IoTDB {} before upgrading to {}.",
- properties.getProperty(IOTDB_VERSION_STRING),
- IoTDBConstant.VERSION);
- System.exit(-1);
- }
- }
-
- private void checkUnClosedTsFileV2InFolders(List<String> folders) {
- for (String baseDir : folders) {
- File fileFolder = fsFactory.getFile(baseDir);
- if (!fileFolder.isDirectory()) {
- continue;
- }
- for (File storageGroup : fileFolder.listFiles()) {
- if (!storageGroup.isDirectory()) {
- continue;
- }
- for (File partitionDir : storageGroup.listFiles()) {
- if (!partitionDir.isDirectory()) {
- continue;
- }
- File[] tsfiles =
- fsFactory.listFilesBySuffix(partitionDir.toString(), TsFileConstant.TSFILE_SUFFIX);
- File[] resources =
- fsFactory.listFilesBySuffix(partitionDir.toString(), TsFileResource.RESOURCE_SUFFIX);
- if (tsfiles.length != resources.length) {
- File[] zeroLevelTsFiles =
- fsFactory.listFilesBySuffix(
- partitionDir.toString(), "0" + TsFileConstant.TSFILE_SUFFIX);
- File[] zeroLevelResources =
- fsFactory.listFilesBySuffix(
- partitionDir.toString(),
- "0" + TsFileConstant.TSFILE_SUFFIX + TsFileResource.RESOURCE_SUFFIX);
- if (zeroLevelTsFiles.length != zeroLevelResources.length) {
- logger.error(
- "Unclosed Version-2 TsFile detected, please stop insertion, then run 'flush' "
- + "on IoTDB {} before upgrading to {}",
- properties.getProperty(IOTDB_VERSION_STRING),
- IoTDBConstant.VERSION);
- System.exit(-1);
- }
- }
- }
- }
- }
- }
-
- /**
- * If upgrading from v0.11.2 to v0.12, there may be some unsealed merging files. We have to delete
- * these files before upgrading.
- */
- private File[] deleteMergingTsFiles(File[] tsfiles, File[] resources) {
- Set<String> resourcesSet = new HashSet<>();
- for (File resource : resources) {
- resourcesSet.add(resource.getName());
- }
- List<File> tsfileList = new ArrayList<>();
- for (File tsfile : tsfiles) {
- if (!resourcesSet.contains(tsfile.getName() + TsFileResource.RESOURCE_SUFFIX)) {
- try {
- logger.info("Delete merging TsFile {}", tsfile);
- Files.delete(tsfile.toPath());
- } catch (Exception e) {
- logger.error("Failed to delete merging tsfile {} ", tsfile, e);
- System.exit(-1);
- }
- } else {
- tsfileList.add(tsfile);
- }
- }
- return tsfileList.toArray(new File[tsfileList.size()]);
- }
-
- private void moveTsFileV2() {
- moveFileToUpgradeFolder(DirectoryManager.getInstance().getAllSequenceFileFolders());
- moveFileToUpgradeFolder(DirectoryManager.getInstance().getAllUnSequenceFileFolders());
- logger.info("Move version-2 TsFile successfully");
- }
-
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- private void moveFileToUpgradeFolder(List<String> folders) {
- for (String baseDir : folders) {
- File fileFolder = fsFactory.getFile(baseDir);
- if (!fileFolder.isDirectory()) {
- continue;
- }
- for (File storageGroup : fileFolder.listFiles()) {
- if (!storageGroup.isDirectory()) {
- continue;
- }
- // create virtual storage group folder 0
- File virtualStorageGroupDir = fsFactory.getFile(storageGroup, "0");
- if (virtualStorageGroupDir.mkdirs()) {
- logger.info(
- "virtual storage directory {} doesn't exist, create it",
- virtualStorageGroupDir.getPath());
- } else if (!virtualStorageGroupDir.exists()) {
- logger.error(
- "Create virtual storage directory {} failed", virtualStorageGroupDir.getPath());
- }
- for (File partitionDir : storageGroup.listFiles()) {
- if (!partitionDir.isDirectory()) {
- continue;
- }
- File[] oldTsfileArray =
- fsFactory.listFilesBySuffix(
- partitionDir.getAbsolutePath(), TsFileConstant.TSFILE_SUFFIX);
- File[] oldResourceFileArray =
- fsFactory.listFilesBySuffix(
- partitionDir.getAbsolutePath(), TsFileResource.RESOURCE_SUFFIX);
- File[] oldModificationFileArray =
- fsFactory.listFilesBySuffix(
- partitionDir.getAbsolutePath(), ModificationFile.FILE_SUFFIX);
- oldTsfileArray = deleteMergingTsFiles(oldTsfileArray, oldResourceFileArray);
- // move the old files to upgrade folder if exists
- if (oldTsfileArray.length + oldResourceFileArray.length + oldModificationFileArray.length
- != 0) {
- // create upgrade directory if not exist
- File upgradeFolder =
- fsFactory.getFile(virtualStorageGroupDir, IoTDBConstant.UPGRADE_FOLDER_NAME);
- if (upgradeFolder.mkdirs()) {
- logger.info("Upgrade Directory {} doesn't exist, create it", upgradeFolder.getPath());
- } else if (!upgradeFolder.exists()) {
- logger.error("Create upgrade Directory {} failed", upgradeFolder.getPath());
- }
- // move .tsfile to upgrade folder
- for (File file : oldTsfileArray) {
- if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
- logger.error("Failed to move tsfile {} to upgrade folder", file);
- }
- }
- // move .resource to upgrade folder
- for (File file : oldResourceFileArray) {
- if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
- logger.error("Failed to move resource {} to upgrade folder", file);
- }
- }
- // move .mods to upgrade folder
- for (File file : oldModificationFileArray) {
- if (!file.renameTo(fsFactory.getFile(upgradeFolder, file.getName()))) {
- logger.error("Failed to move mod file {} to upgrade folder", file);
- }
- }
- }
- if (partitionDir.listFiles().length == 0) {
- try {
- Files.delete(partitionDir.toPath());
- } catch (IOException e) {
- logger.error("Delete {} failed", partitionDir);
- }
- }
- }
- }
- }
- }
-
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
- private void moveVersionFile() {
- File sgDir =
- SystemFileFactory.INSTANCE.getFile(
- FilePathUtils.regularizePath(SYSTEM_DIR) + "storage_groups");
- if (sgDir.isDirectory()) {
- for (File sg : sgDir.listFiles()) {
- if (!sg.isDirectory()) {
- continue;
- }
- for (File partition : sg.listFiles()) {
- if (!partition.isDirectory()) {
- continue;
- }
- File virtualSg = SystemFileFactory.INSTANCE.getFile(sg, "0");
- if (!virtualSg.exists()) {
- virtualSg.mkdir();
- }
- File newPartition = SystemFileFactory.INSTANCE.getFile(virtualSg, partition.getName());
- if (!newPartition.exists()) {
- newPartition.mkdir();
- }
- for (File versionFile : partition.listFiles()) {
- if (versionFile.isDirectory()) {
- continue;
- }
- if (!versionFile.renameTo(
- SystemFileFactory.INSTANCE.getFile(newPartition, versionFile.getName()))) {
- logger.error("Rename {} failed", versionFile);
- }
- }
- if (partition.listFiles().length == 0) {
- try {
- Files.delete(partition.toPath());
- } catch (IOException e) {
- logger.error("Delete {} failed", partition);
- }
- }
- }
- }
- }
- }
}