You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@iotdb.apache.org by GitBox <gi...@apache.org> on 2020/05/24 06:37:55 UTC
[GitHub] [incubator-iotdb] HTHou commented on a change in pull request #1248: [IOTDB-697] add enablePartition in config
HTHou commented on a change in pull request #1248:
URL: https://github.com/apache/incubator-iotdb/pull/1248#discussion_r429604483
##########
File path: server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
##########
@@ -37,182 +40,242 @@
public class IoTDBConfigCheck {
+ private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
+
// this file is located in data/system/schema/system.properties
// If user delete folder "data", system.properties can reset.
- public static final String PROPERTIES_FILE_NAME = "system.properties";
- public static final String SCHEMA_DIR =
- IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
- public static final String WAL_DIR =
- IoTDBDescriptor.getInstance().getConfig().getWalFolder();
- private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
- // this is a initial parameter.
- private static String timestampPrecision = "ms";
- private static long partitionInterval = 86400;
- private static String tsfileFileSystem = "LOCAL";
- private static String iotdbVersion = "0.10.0";
+ private static final String PROPERTIES_FILE_NAME = "system.properties";
+ private static final String SCHEMA_DIR = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+ private static final String WAL_DIR = IoTDBDescriptor.getInstance().getConfig().getWalFolder();
+
+ File propertiesFile;
+ File tmpPropertiesFile;
+
private Properties properties = new Properties();
+ private Map<String, String> systemProperties = new HashMap<>();
+
+ private static final String SYSTEM_PROPERTIES_STRING = "System properties:";
+
+ private static final String TIMESTAMP_PRECISION_STRING = "timestamp_precision";
+ private static String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+
+ private static final String PARTITION_INTERVAL_STRING = "partition_interval";
+ private static long partitionInterval = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
+
+ private static final String TSFILE_FILE_SYSTEM_STRING = "tsfile_storage_fs";
+ private static String tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
+
+ private static final String ENABLE_PARTITION_STRING = "enable_partition";
+ private static boolean enablePartition = IoTDBDescriptor.getInstance().getConfig().isEnablePartition();
+
+ private static final String IOTDB_VERSION_STRING = "iotdb_version";
+ private static String iotdbVersion = "0.10.0";
+
public static IoTDBConfigCheck getInstance() {
return IoTDBConfigCheckHolder.INSTANCE;
}
- public void checkConfig() {
- timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ private static class IoTDBConfigCheckHolder {
+ private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
+ }
+
+ private IoTDBConfigCheck() {
+ logger.info("Starting IoTDB " + iotdbVersion);
+
+ // check whether SCHEMA_DIR exists, create if not exists
+ File dir = SystemFileFactory.INSTANCE.getFile(SCHEMA_DIR);
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ logger.error("can not create schema dir: {}", SCHEMA_DIR);
+ System.exit(-1);
+ } else {
+ logger.info(" {} dir has been created.", SCHEMA_DIR);
+ }
+ }
// check time stamp precision
if (!(timestampPrecision.equals("ms") || timestampPrecision.equals("us")
- || timestampPrecision.equals("ns"))) {
- logger.error("Wrong timestamp precision, please set as: ms, us or ns ! Current is: "
- + timestampPrecision);
+ || timestampPrecision.equals("ns"))) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: ms, us or ns ! Current is: "
+ + timestampPrecision);
System.exit(-1);
}
- partitionInterval = IoTDBDescriptor.getInstance().getConfig()
- .getPartitionInterval();
+ if (!enablePartition) {
+ partitionInterval = Long.MAX_VALUE;
+ }
// check partition interval
if (partitionInterval <= 0) {
logger.error("Partition interval must larger than 0!");
System.exit(-1);
}
- tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
- createDir(SCHEMA_DIR);
- checkFile(SCHEMA_DIR);
- logger.info("System configuration is ok.");
-
+ systemProperties.put(TIMESTAMP_PRECISION_STRING, timestampPrecision);
+ systemProperties.put(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ systemProperties.put(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ systemProperties.put(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ systemProperties.put(IOTDB_VERSION_STRING, iotdbVersion);
}
- private void createDir(String filepath) {
- File dir = SystemFileFactory.INSTANCE.getFile(filepath);
- if (!dir.exists()) {
- dir.mkdirs();
- logger.info(" {} dir has been created.", SCHEMA_DIR);
- }
- }
- private void checkFile(String filepath) {
- // create file : read timestamp precision from engine.properties, create system.properties
- // use output stream to write timestamp precision to file.
- File file = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- File tmpPropertiesFile = new File(file.getAbsoluteFile() + ".tmp");
- try {
- if (!file.exists() && !tmpPropertiesFile.exists()) {
- file.createNewFile();
- logger.info(" {} has been created.", file.getAbsolutePath());
- try (FileOutputStream outputStream = new FileOutputStream(file.toString())) {
- properties.setProperty("timestamp_precision", timestampPrecision);
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- }
- return;
+ /**
+ * check configuration in system.properties when starting IoTDB
+ *
+ * When init: create system.properties directly
+ *
+ * When upgrading the system.properties:
+ * (1) create system.properties.tmp
+ * (2) delete system.properties
+ * (2) rename system.properties.tmp to system.properties
+ */
+ public void checkConfig() throws IOException {
+ propertiesFile = SystemFileFactory.INSTANCE
+ .getFile(IoTDBConfigCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME);
+ tmpPropertiesFile = new File(propertiesFile.getAbsoluteFile() + ".tmp");
+
+ // system init first time, no need to check, write system.properties and return
+ if (!propertiesFile.exists() && !tmpPropertiesFile.exists()) {
+ // create system.properties
+ if (propertiesFile.createNewFile()) {
+ logger.info(" {} has been created.", propertiesFile.getAbsolutePath());
+ } else {
+ logger.error("can not create {}", propertiesFile.getAbsolutePath());
+ System.exit(-1);
}
- else if (!file.exists() && tmpPropertiesFile.exists()) {
- // rename upgraded system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- logger.info(" {} has been upgraded.", file.getAbsolutePath());
- checkProperties();
- return;
+
+ // write properties to system.properties
+ try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+ properties.store(outputStream, SYSTEM_PROPERTIES_STRING);
}
- } catch (IOException e) {
- logger.error("Can not create {}.", file.getAbsolutePath(), e);
- }
-
- // get existed properties from system_properties.txt
- File inputFile = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
+ return;
+ }
+
+ if (!propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // rename tmp file to system.properties, no need to check
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
+ logger.info("rename {} to {}", tmpPropertiesFile, propertiesFile);
+ return;
+ } else if (propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // both files exist, remove tmp file
+ FileUtils.forceDeleteOnExit(tmpPropertiesFile);
+ logger.info("remove {}", tmpPropertiesFile);
+ }
+
+ // no tmp file, read properties from system.properties
+ try (FileInputStream inputStream = new FileInputStream(propertiesFile)) {
properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET));
- // need to upgrade
- if (!properties.containsKey("iotdb_version")) {
+ // need to upgrade from 0.9 to 0.10
+ if (!properties.containsKey(IOTDB_VERSION_STRING)) {
checkUnClosedTsFileV1();
- upgradeMlog();
- } else {
- checkProperties();
- return;
+ MLogWriter.upgradeMLog(SCHEMA_DIR, MetadataConstant.METADATA_LOG);
+ upgradePropertiesFile();
}
- } catch (IOException e) {
- logger.error("Load system.properties from {} failed.", file.getAbsolutePath(), e);
+ checkProperties();
+ }
+ }
+
+ /**
+ * upgrade 0.9 properties to 0.10 properties
+ */
+ private void upgradePropertiesFile()
+ throws IOException {
+ // create an empty tmpPropertiesFile
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
}
- // if tmpPropertiesFile exists, remove it
- if (tmpPropertiesFile.exists()) {
- try {
- Files.delete(tmpPropertiesFile.toPath());
- } catch (IOException e) {
- logger.error("Fail to remove broken file {}", tmpPropertiesFile);
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ properties.setProperty(IOTDB_VERSION_STRING, iotdbVersion);
+ properties.setProperty(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
+
+ // upgrade finished, delete old system.properties file
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
+ // rename system.properties.tmp to system.properties
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
+ }
+
+
+ /**
+ * repair 0.10 properties
+ */
+ private void upgradePropertiesFileFromBrokenFile()
+ throws IOException {
// create an empty tmpPropertiesFile
- try {
- if (tmpPropertiesFile.createNewFile()) {
- logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
- }
- } catch (IOException e) {
- logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile, e);
- }
- // try to add the storage_group_time_range, tsfile_storage_fs
- // and iotdb_version property in system.properties.tmp
- try (FileOutputStream outputStream = new FileOutputStream(tmpPropertiesFile.toString())) {
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- checkProperties();
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
+ }
+
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
// upgrade finished, delete old system.properties file
- if (file.exists()) {
- Files.delete(file.toPath());
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
// rename system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- } catch (IOException e) {
- logger.error("Something went wrong while upgrading teh system.properties. The file is {}.", file.getAbsolutePath(), e);
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
-
}
- private void checkProperties() {
- if (!properties.getProperty("timestamp_precision").equals(timestampPrecision)) {
- logger.error("Wrong timestamp precision, please set as: " + properties
- .getProperty("timestamp_precision") + " !");
- System.exit(-1);
+ private void checkProperties() throws IOException {
+ for (Entry<String, String> entry : systemProperties.entrySet()) {
+ if (!properties.contains(entry.getKey())) {
+ upgradePropertiesFileFromBrokenFile();
+ logger.info("repair system.properties, lack {}", entry.getKey());
+ }
}
- if (!(Long.parseLong(properties.getProperty("storage_group_time_range"))
- == partitionInterval)) {
- logger.error("Wrong storage group time range, please set as: " + properties
- .getProperty("storage_group_time_range") + " !");
+
+ if (!properties.getProperty(TIMESTAMP_PRECISION_STRING).equals(timestampPrecision)) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: " + properties
+ .getProperty(TIMESTAMP_PRECISION_STRING) + " !");
System.exit(-1);
}
- if (!(properties.getProperty("tsfile_storage_fs").equals(tsfileFileSystem))) {
- logger.error("Wrong tsfile file system, please set as: " + properties
- .getProperty("tsfile_storage_fs") + " !");
+
+ if (!(Long.parseLong(properties.getProperty(PARTITION_INTERVAL_STRING))
+ == partitionInterval)) {
+ logger.error("Wrong " + PARTITION_INTERVAL_STRING + ", please set as: " + properties
+ .getProperty(PARTITION_INTERVAL_STRING) + " !");
System.exit(-1);
}
- }
- private void upgradeMlog() {
- try {
- MLogWriter.upgradeMLog(SCHEMA_DIR, MetadataConstant.METADATA_LOG);
- } catch (IOException e) {
- logger.error("Upgrade mlog.txt from {} failed.", SCHEMA_DIR, e);
+ if (!(properties.getProperty(TSFILE_FILE_SYSTEM_STRING).equals(tsfileFileSystem))) {
+ logger.error("Wrong " + TSFILE_FILE_SYSTEM_STRING + ", please set as: " + properties
+ .getProperty(TSFILE_FILE_SYSTEM_STRING) + " !");
Review comment:
```suggestion
logger.error("Wrong {}, please set as: {} !", TSFILE_FILE_SYSTEM_STRING, properties
.getProperty(TSFILE_FILE_SYSTEM_STRING));
```
##########
File path: server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
##########
@@ -37,182 +40,242 @@
public class IoTDBConfigCheck {
+ private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
+
// this file is located in data/system/schema/system.properties
// If user delete folder "data", system.properties can reset.
- public static final String PROPERTIES_FILE_NAME = "system.properties";
- public static final String SCHEMA_DIR =
- IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
- public static final String WAL_DIR =
- IoTDBDescriptor.getInstance().getConfig().getWalFolder();
- private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
- // this is a initial parameter.
- private static String timestampPrecision = "ms";
- private static long partitionInterval = 86400;
- private static String tsfileFileSystem = "LOCAL";
- private static String iotdbVersion = "0.10.0";
+ private static final String PROPERTIES_FILE_NAME = "system.properties";
+ private static final String SCHEMA_DIR = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+ private static final String WAL_DIR = IoTDBDescriptor.getInstance().getConfig().getWalFolder();
+
+ File propertiesFile;
+ File tmpPropertiesFile;
+
private Properties properties = new Properties();
+ private Map<String, String> systemProperties = new HashMap<>();
+
+ private static final String SYSTEM_PROPERTIES_STRING = "System properties:";
+
+ private static final String TIMESTAMP_PRECISION_STRING = "timestamp_precision";
+ private static String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+
+ private static final String PARTITION_INTERVAL_STRING = "partition_interval";
+ private static long partitionInterval = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
+
+ private static final String TSFILE_FILE_SYSTEM_STRING = "tsfile_storage_fs";
+ private static String tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
+
+ private static final String ENABLE_PARTITION_STRING = "enable_partition";
+ private static boolean enablePartition = IoTDBDescriptor.getInstance().getConfig().isEnablePartition();
+
+ private static final String IOTDB_VERSION_STRING = "iotdb_version";
+ private static String iotdbVersion = "0.10.0";
+
public static IoTDBConfigCheck getInstance() {
return IoTDBConfigCheckHolder.INSTANCE;
}
- public void checkConfig() {
- timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ private static class IoTDBConfigCheckHolder {
+ private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
+ }
+
+ private IoTDBConfigCheck() {
+ logger.info("Starting IoTDB " + iotdbVersion);
+
+ // check whether SCHEMA_DIR exists, create if not exists
+ File dir = SystemFileFactory.INSTANCE.getFile(SCHEMA_DIR);
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ logger.error("can not create schema dir: {}", SCHEMA_DIR);
+ System.exit(-1);
+ } else {
+ logger.info(" {} dir has been created.", SCHEMA_DIR);
+ }
+ }
// check time stamp precision
if (!(timestampPrecision.equals("ms") || timestampPrecision.equals("us")
- || timestampPrecision.equals("ns"))) {
- logger.error("Wrong timestamp precision, please set as: ms, us or ns ! Current is: "
- + timestampPrecision);
+ || timestampPrecision.equals("ns"))) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: ms, us or ns ! Current is: "
+ + timestampPrecision);
System.exit(-1);
}
- partitionInterval = IoTDBDescriptor.getInstance().getConfig()
- .getPartitionInterval();
+ if (!enablePartition) {
+ partitionInterval = Long.MAX_VALUE;
+ }
// check partition interval
if (partitionInterval <= 0) {
logger.error("Partition interval must larger than 0!");
System.exit(-1);
}
- tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
- createDir(SCHEMA_DIR);
- checkFile(SCHEMA_DIR);
- logger.info("System configuration is ok.");
-
+ systemProperties.put(TIMESTAMP_PRECISION_STRING, timestampPrecision);
+ systemProperties.put(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ systemProperties.put(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ systemProperties.put(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ systemProperties.put(IOTDB_VERSION_STRING, iotdbVersion);
}
- private void createDir(String filepath) {
- File dir = SystemFileFactory.INSTANCE.getFile(filepath);
- if (!dir.exists()) {
- dir.mkdirs();
- logger.info(" {} dir has been created.", SCHEMA_DIR);
- }
- }
- private void checkFile(String filepath) {
- // create file : read timestamp precision from engine.properties, create system.properties
- // use output stream to write timestamp precision to file.
- File file = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- File tmpPropertiesFile = new File(file.getAbsoluteFile() + ".tmp");
- try {
- if (!file.exists() && !tmpPropertiesFile.exists()) {
- file.createNewFile();
- logger.info(" {} has been created.", file.getAbsolutePath());
- try (FileOutputStream outputStream = new FileOutputStream(file.toString())) {
- properties.setProperty("timestamp_precision", timestampPrecision);
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- }
- return;
+ /**
+ * check configuration in system.properties when starting IoTDB
+ *
+ * When init: create system.properties directly
+ *
+ * When upgrading the system.properties:
+ * (1) create system.properties.tmp
+ * (2) delete system.properties
+ * (2) rename system.properties.tmp to system.properties
+ */
+ public void checkConfig() throws IOException {
+ propertiesFile = SystemFileFactory.INSTANCE
+ .getFile(IoTDBConfigCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME);
+ tmpPropertiesFile = new File(propertiesFile.getAbsoluteFile() + ".tmp");
+
+ // system init first time, no need to check, write system.properties and return
+ if (!propertiesFile.exists() && !tmpPropertiesFile.exists()) {
+ // create system.properties
+ if (propertiesFile.createNewFile()) {
+ logger.info(" {} has been created.", propertiesFile.getAbsolutePath());
+ } else {
+ logger.error("can not create {}", propertiesFile.getAbsolutePath());
+ System.exit(-1);
}
- else if (!file.exists() && tmpPropertiesFile.exists()) {
- // rename upgraded system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- logger.info(" {} has been upgraded.", file.getAbsolutePath());
- checkProperties();
- return;
+
+ // write properties to system.properties
+ try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+ properties.store(outputStream, SYSTEM_PROPERTIES_STRING);
}
- } catch (IOException e) {
- logger.error("Can not create {}.", file.getAbsolutePath(), e);
- }
-
- // get existed properties from system_properties.txt
- File inputFile = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
+ return;
+ }
+
+ if (!propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // rename tmp file to system.properties, no need to check
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
+ logger.info("rename {} to {}", tmpPropertiesFile, propertiesFile);
+ return;
+ } else if (propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // both files exist, remove tmp file
+ FileUtils.forceDeleteOnExit(tmpPropertiesFile);
+ logger.info("remove {}", tmpPropertiesFile);
+ }
+
+ // no tmp file, read properties from system.properties
+ try (FileInputStream inputStream = new FileInputStream(propertiesFile)) {
properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET));
- // need to upgrade
- if (!properties.containsKey("iotdb_version")) {
+ // need to upgrade from 0.9 to 0.10
+ if (!properties.containsKey(IOTDB_VERSION_STRING)) {
checkUnClosedTsFileV1();
- upgradeMlog();
- } else {
- checkProperties();
- return;
+ MLogWriter.upgradeMLog(SCHEMA_DIR, MetadataConstant.METADATA_LOG);
+ upgradePropertiesFile();
}
- } catch (IOException e) {
- logger.error("Load system.properties from {} failed.", file.getAbsolutePath(), e);
+ checkProperties();
+ }
+ }
+
+ /**
+ * upgrade 0.9 properties to 0.10 properties
+ */
+ private void upgradePropertiesFile()
+ throws IOException {
+ // create an empty tmpPropertiesFile
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
}
- // if tmpPropertiesFile exists, remove it
- if (tmpPropertiesFile.exists()) {
- try {
- Files.delete(tmpPropertiesFile.toPath());
- } catch (IOException e) {
- logger.error("Fail to remove broken file {}", tmpPropertiesFile);
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ properties.setProperty(IOTDB_VERSION_STRING, iotdbVersion);
+ properties.setProperty(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
+
+ // upgrade finished, delete old system.properties file
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
+ // rename system.properties.tmp to system.properties
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
+ }
+
+
+ /**
+ * repair 0.10 properties
+ */
+ private void upgradePropertiesFileFromBrokenFile()
+ throws IOException {
// create an empty tmpPropertiesFile
- try {
- if (tmpPropertiesFile.createNewFile()) {
- logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
- }
- } catch (IOException e) {
- logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile, e);
- }
- // try to add the storage_group_time_range, tsfile_storage_fs
- // and iotdb_version property in system.properties.tmp
- try (FileOutputStream outputStream = new FileOutputStream(tmpPropertiesFile.toString())) {
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- checkProperties();
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
+ }
+
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
// upgrade finished, delete old system.properties file
- if (file.exists()) {
- Files.delete(file.toPath());
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
// rename system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- } catch (IOException e) {
- logger.error("Something went wrong while upgrading teh system.properties. The file is {}.", file.getAbsolutePath(), e);
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
-
}
- private void checkProperties() {
- if (!properties.getProperty("timestamp_precision").equals(timestampPrecision)) {
- logger.error("Wrong timestamp precision, please set as: " + properties
- .getProperty("timestamp_precision") + " !");
- System.exit(-1);
+ private void checkProperties() throws IOException {
+ for (Entry<String, String> entry : systemProperties.entrySet()) {
+ if (!properties.contains(entry.getKey())) {
+ upgradePropertiesFileFromBrokenFile();
+ logger.info("repair system.properties, lack {}", entry.getKey());
+ }
}
- if (!(Long.parseLong(properties.getProperty("storage_group_time_range"))
- == partitionInterval)) {
- logger.error("Wrong storage group time range, please set as: " + properties
- .getProperty("storage_group_time_range") + " !");
+
+ if (!properties.getProperty(TIMESTAMP_PRECISION_STRING).equals(timestampPrecision)) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: " + properties
+ .getProperty(TIMESTAMP_PRECISION_STRING) + " !");
System.exit(-1);
}
- if (!(properties.getProperty("tsfile_storage_fs").equals(tsfileFileSystem))) {
- logger.error("Wrong tsfile file system, please set as: " + properties
- .getProperty("tsfile_storage_fs") + " !");
+
+ if (!(Long.parseLong(properties.getProperty(PARTITION_INTERVAL_STRING))
+ == partitionInterval)) {
Review comment:
```suggestion
if (Long.parseLong(properties.getProperty(PARTITION_INTERVAL_STRING))
!= partitionInterval) {
```
##########
File path: server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
##########
@@ -37,182 +40,242 @@
public class IoTDBConfigCheck {
+ private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
+
// this file is located in data/system/schema/system.properties
// If user delete folder "data", system.properties can reset.
- public static final String PROPERTIES_FILE_NAME = "system.properties";
- public static final String SCHEMA_DIR =
- IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
- public static final String WAL_DIR =
- IoTDBDescriptor.getInstance().getConfig().getWalFolder();
- private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
- // this is a initial parameter.
- private static String timestampPrecision = "ms";
- private static long partitionInterval = 86400;
- private static String tsfileFileSystem = "LOCAL";
- private static String iotdbVersion = "0.10.0";
+ private static final String PROPERTIES_FILE_NAME = "system.properties";
+ private static final String SCHEMA_DIR = IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+ private static final String WAL_DIR = IoTDBDescriptor.getInstance().getConfig().getWalFolder();
+
+ File propertiesFile;
+ File tmpPropertiesFile;
+
private Properties properties = new Properties();
+ private Map<String, String> systemProperties = new HashMap<>();
+
+ private static final String SYSTEM_PROPERTIES_STRING = "System properties:";
+
+ private static final String TIMESTAMP_PRECISION_STRING = "timestamp_precision";
+ private static String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+
+ private static final String PARTITION_INTERVAL_STRING = "partition_interval";
+ private static long partitionInterval = IoTDBDescriptor.getInstance().getConfig().getPartitionInterval();
+
+ private static final String TSFILE_FILE_SYSTEM_STRING = "tsfile_storage_fs";
+ private static String tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
+
+ private static final String ENABLE_PARTITION_STRING = "enable_partition";
+ private static boolean enablePartition = IoTDBDescriptor.getInstance().getConfig().isEnablePartition();
+
+ private static final String IOTDB_VERSION_STRING = "iotdb_version";
+ private static String iotdbVersion = "0.10.0";
+
public static IoTDBConfigCheck getInstance() {
return IoTDBConfigCheckHolder.INSTANCE;
}
- public void checkConfig() {
- timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ private static class IoTDBConfigCheckHolder {
+ private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
+ }
+
+ private IoTDBConfigCheck() {
+ logger.info("Starting IoTDB " + iotdbVersion);
+
+ // check whether SCHEMA_DIR exists, create if not exists
+ File dir = SystemFileFactory.INSTANCE.getFile(SCHEMA_DIR);
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ logger.error("can not create schema dir: {}", SCHEMA_DIR);
+ System.exit(-1);
+ } else {
+ logger.info(" {} dir has been created.", SCHEMA_DIR);
+ }
+ }
// check time stamp precision
if (!(timestampPrecision.equals("ms") || timestampPrecision.equals("us")
- || timestampPrecision.equals("ns"))) {
- logger.error("Wrong timestamp precision, please set as: ms, us or ns ! Current is: "
- + timestampPrecision);
+ || timestampPrecision.equals("ns"))) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: ms, us or ns ! Current is: "
+ + timestampPrecision);
System.exit(-1);
}
- partitionInterval = IoTDBDescriptor.getInstance().getConfig()
- .getPartitionInterval();
+ if (!enablePartition) {
+ partitionInterval = Long.MAX_VALUE;
+ }
// check partition interval
if (partitionInterval <= 0) {
logger.error("Partition interval must larger than 0!");
System.exit(-1);
}
- tsfileFileSystem = IoTDBDescriptor.getInstance().getConfig().getTsFileStorageFs().toString();
- createDir(SCHEMA_DIR);
- checkFile(SCHEMA_DIR);
- logger.info("System configuration is ok.");
-
+ systemProperties.put(TIMESTAMP_PRECISION_STRING, timestampPrecision);
+ systemProperties.put(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ systemProperties.put(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ systemProperties.put(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ systemProperties.put(IOTDB_VERSION_STRING, iotdbVersion);
}
- private void createDir(String filepath) {
- File dir = SystemFileFactory.INSTANCE.getFile(filepath);
- if (!dir.exists()) {
- dir.mkdirs();
- logger.info(" {} dir has been created.", SCHEMA_DIR);
- }
- }
- private void checkFile(String filepath) {
- // create file : read timestamp precision from engine.properties, create system.properties
- // use output stream to write timestamp precision to file.
- File file = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- File tmpPropertiesFile = new File(file.getAbsoluteFile() + ".tmp");
- try {
- if (!file.exists() && !tmpPropertiesFile.exists()) {
- file.createNewFile();
- logger.info(" {} has been created.", file.getAbsolutePath());
- try (FileOutputStream outputStream = new FileOutputStream(file.toString())) {
- properties.setProperty("timestamp_precision", timestampPrecision);
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- }
- return;
+ /**
+ * check configuration in system.properties when starting IoTDB
+ *
+ * When init: create system.properties directly
+ *
+ * When upgrading the system.properties:
+ * (1) create system.properties.tmp
+ * (2) delete system.properties
+ * (2) rename system.properties.tmp to system.properties
+ */
+ public void checkConfig() throws IOException {
+ propertiesFile = SystemFileFactory.INSTANCE
+ .getFile(IoTDBConfigCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME);
+ tmpPropertiesFile = new File(propertiesFile.getAbsoluteFile() + ".tmp");
+
+ // system init first time, no need to check, write system.properties and return
+ if (!propertiesFile.exists() && !tmpPropertiesFile.exists()) {
+ // create system.properties
+ if (propertiesFile.createNewFile()) {
+ logger.info(" {} has been created.", propertiesFile.getAbsolutePath());
+ } else {
+ logger.error("can not create {}", propertiesFile.getAbsolutePath());
+ System.exit(-1);
}
- else if (!file.exists() && tmpPropertiesFile.exists()) {
- // rename upgraded system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- logger.info(" {} has been upgraded.", file.getAbsolutePath());
- checkProperties();
- return;
+
+ // write properties to system.properties
+ try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+ properties.store(outputStream, SYSTEM_PROPERTIES_STRING);
}
- } catch (IOException e) {
- logger.error("Can not create {}.", file.getAbsolutePath(), e);
- }
-
- // get existed properties from system_properties.txt
- File inputFile = SystemFileFactory.INSTANCE
- .getFile(filepath + File.separator + PROPERTIES_FILE_NAME);
- try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
+ return;
+ }
+
+ if (!propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // rename tmp file to system.properties, no need to check
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
+ logger.info("rename {} to {}", tmpPropertiesFile, propertiesFile);
+ return;
+ } else if (propertiesFile.exists() && tmpPropertiesFile.exists()) {
+ // both files exist, remove tmp file
+ FileUtils.forceDeleteOnExit(tmpPropertiesFile);
+ logger.info("remove {}", tmpPropertiesFile);
+ }
+
+ // no tmp file, read properties from system.properties
+ try (FileInputStream inputStream = new FileInputStream(propertiesFile)) {
properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET));
- // need to upgrade
- if (!properties.containsKey("iotdb_version")) {
+ // need to upgrade from 0.9 to 0.10
+ if (!properties.containsKey(IOTDB_VERSION_STRING)) {
checkUnClosedTsFileV1();
- upgradeMlog();
- } else {
- checkProperties();
- return;
+ MLogWriter.upgradeMLog(SCHEMA_DIR, MetadataConstant.METADATA_LOG);
+ upgradePropertiesFile();
}
- } catch (IOException e) {
- logger.error("Load system.properties from {} failed.", file.getAbsolutePath(), e);
+ checkProperties();
+ }
+ }
+
+ /**
+ * upgrade 0.9 properties to 0.10 properties
+ */
+ private void upgradePropertiesFile()
+ throws IOException {
+ // create an empty tmpPropertiesFile
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
}
- // if tmpPropertiesFile exists, remove it
- if (tmpPropertiesFile.exists()) {
- try {
- Files.delete(tmpPropertiesFile.toPath());
- } catch (IOException e) {
- logger.error("Fail to remove broken file {}", tmpPropertiesFile);
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ properties.setProperty(PARTITION_INTERVAL_STRING, String.valueOf(partitionInterval));
+ properties.setProperty(TSFILE_FILE_SYSTEM_STRING, tsfileFileSystem);
+ properties.setProperty(IOTDB_VERSION_STRING, iotdbVersion);
+ properties.setProperty(ENABLE_PARTITION_STRING, String.valueOf(enablePartition));
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
+
+ // upgrade finished, delete old system.properties file
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
+ // rename system.properties.tmp to system.properties
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
+ }
+
+
+ /**
+ * repair 0.10 properties
+ */
+ private void upgradePropertiesFileFromBrokenFile()
+ throws IOException {
// create an empty tmpPropertiesFile
- try {
- if (tmpPropertiesFile.createNewFile()) {
- logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
- }
- } catch (IOException e) {
- logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile, e);
- }
- // try to add the storage_group_time_range, tsfile_storage_fs
- // and iotdb_version property in system.properties.tmp
- try (FileOutputStream outputStream = new FileOutputStream(tmpPropertiesFile.toString())) {
- properties.setProperty("storage_group_time_range", String.valueOf(partitionInterval));
- properties.setProperty("tsfile_storage_fs", tsfileFileSystem);
- properties.setProperty("iotdb_version", iotdbVersion);
- properties.store(outputStream, "System properties:");
- checkProperties();
+ if (tmpPropertiesFile.createNewFile()) {
+ logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
+ } else {
+ logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
+ System.exit(-1);
+ }
+
+ try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v));
+
+ properties.store(tmpFOS, SYSTEM_PROPERTIES_STRING);
// upgrade finished, delete old system.properties file
- if (file.exists()) {
- Files.delete(file.toPath());
+ if (propertiesFile.exists()) {
+ Files.delete(propertiesFile.toPath());
}
// rename system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, file);
- } catch (IOException e) {
- logger.error("Something went wrong while upgrading teh system.properties. The file is {}.", file.getAbsolutePath(), e);
+ FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
}
-
}
- private void checkProperties() {
- if (!properties.getProperty("timestamp_precision").equals(timestampPrecision)) {
- logger.error("Wrong timestamp precision, please set as: " + properties
- .getProperty("timestamp_precision") + " !");
- System.exit(-1);
+ private void checkProperties() throws IOException {
+ for (Entry<String, String> entry : systemProperties.entrySet()) {
+ if (!properties.contains(entry.getKey())) {
+ upgradePropertiesFileFromBrokenFile();
+ logger.info("repair system.properties, lack {}", entry.getKey());
+ }
}
- if (!(Long.parseLong(properties.getProperty("storage_group_time_range"))
- == partitionInterval)) {
- logger.error("Wrong storage group time range, please set as: " + properties
- .getProperty("storage_group_time_range") + " !");
+
+ if (!properties.getProperty(TIMESTAMP_PRECISION_STRING).equals(timestampPrecision)) {
+ logger.error("Wrong " + TIMESTAMP_PRECISION_STRING + ", please set as: " + properties
+ .getProperty(TIMESTAMP_PRECISION_STRING) + " !");
Review comment:
```suggestion
logger.error("Wrong {}, please set as: {} !", TIMESTAMP_PRECISION_STRING, properties
.getProperty(TIMESTAMP_PRECISION_STRING));
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org