You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/08/25 01:14:50 UTC
[iotdb] branch ExternalLib013 created (now 727fd206b8)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a change to branch ExternalLib013
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 727fd206b8 Support loading external lib in 0.13
This branch includes the following new commits:
new 727fd206b8 Support loading external lib in 0.13
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Support loading external lib in 0.13
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch ExternalLib013
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 727fd206b88eb9d73abee57f4da9f730304f7b98
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Thu Aug 25 09:14:36 2022 +0800
Support loading external lib in 0.13
---
external-api/pom.xml | 60 +
.../iotdb/external/api/IPropertiesLoader.java | 37 +
.../iotdb/external/api/ISeriesNumerLimiter.java | 48 +
pom.xml | 1 +
server/pom.xml | 5 +
.../resources/conf/iotdb-engine.properties | 25 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 29 +
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 1428 ++++++++++----------
.../metadata/SeriesNumberOverflowException.java | 28 +
.../org/apache/iotdb/db/metadata/MManager.java | 191 ++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +
.../org/apache/iotdb/db/utils/JarLoaderUtil.java | 156 +++
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 2 +
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 2 +-
15 files changed, 1268 insertions(+), 753 deletions(-)
diff --git a/external-api/pom.xml b/external-api/pom.xml
new file mode 100644
index 0000000000..dd28dcdc69
--- /dev/null
+++ b/external-api/pom.xml
@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>iotdb-parent</artifactId>
+ <groupId>org.apache.iotdb</groupId>
+ <version>0.13.2-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>external-api</artifactId>
+ <profiles>
+ <profile>
+ <id>get-jar-with-dependencies</id>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.version}</version>
+ <configuration>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <executions>
+ <execution>
+ <id>make-assembly</id>
+ <!-- this is used for inheritance merges -->
+ <phase>package</phase>
+ <!-- bind to the packaging phase -->
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java
new file mode 100644
index 0000000000..10c5b21e7e
--- /dev/null
+++ b/external-api/src/main/java/org/apche/iotdb/external/api/IPropertiesLoader.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apche.iotdb.external.api;
+
+import java.nio.file.Path;
+import java.util.Properties;
+
+/**
+ * An interface to load properties from external properties file to override the default
+ * configurations
+ */
+public interface IPropertiesLoader {
+
+ /**
+ * Load Properties from specific file
+ *
+ * @param file The path of the properties file to open
+ * @return a property list with values in file.
+ */
+ Properties loadProperties(Path file);
+}
diff --git a/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java
new file mode 100644
index 0000000000..7b36b5d8b7
--- /dev/null
+++ b/external-api/src/main/java/org/apche/iotdb/external/api/ISeriesNumerLimiter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apche.iotdb.external.api;
+
+import java.util.Properties;
+
+/** An interface for series number limiting, users can implement their own limitation strategy */
+public interface ISeriesNumerLimiter {
+
+ /**
+ * do the necessary initialization
+ *
+ * @param properties Properties containing all the parameters needed to init
+ */
+ void init(Properties properties);
+
+ /**
+ * add time series
+ *
+ * @param number time series number for current createTimeSeries operation
+ * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries
+ * operation is allowed, otherwise false
+ */
+ boolean addTimeSeries(int number);
+
+ /**
+ * delete time series
+ *
+ * @param number time series number for current deleteTimeSeries operation
+ */
+ void deleteTimeSeries(int number);
+}
diff --git a/pom.xml b/pom.xml
index 31d9b94d56..0b742019b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,6 +110,7 @@
<module>metrics</module>
<module>integration</module>
<module>rewriteFileTool</module>
+ <module>external-api</module>
<!-- <module>library-udf</module>-->
</modules>
<!-- Properties Management -->
diff --git a/server/pom.xml b/server/pom.xml
index cadf11281e..a8cdc0088a 100644
--- a/server/pom.xml
+++ b/server/pom.xml
@@ -67,6 +67,11 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.iotdb</groupId>
+ <artifactId>external-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.iotdb</groupId>
<artifactId>iotdb-thrift-sync</artifactId>
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 0be00a433d..ebd1a35242 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -935,4 +935,27 @@ timestamp_precision=ms
### Group By Fill Configuration
####################
# Datatype: float
-# group_by_fill_cache_size_in_mb=1.0
\ No newline at end of file
+# group_by_fill_cache_size_in_mb=1.0
+
+
+####################
+### External Lib Configuration
+####################
+
+# external lib directory for properties loader
+# For Window platform
+# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
+# absolute. Otherwise, it is relative.
+# external_properties_loader_dir=ext\\loader
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# external_properties_loader_dir=ext/loader
+
+# external lib directory for limiter
+# For Window platform
+# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
+# absolute. Otherwise, it is relative.
+# external_limiter_dir=ext\\limiter
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# external_limiter_dir=ext/limiter
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 997ea249d6..c3e0bdcde9 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -49,6 +49,7 @@ public class IoTDBConfig {
/* Names of Watermark methods */
public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod";
static final String CONFIG_NAME = "iotdb-engine.properties";
+ public static final String EXTERNAL_CONFIG_NAME = "iotdb-engine-external.properties";
private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class);
private static final String MULTI_DIR_STRATEGY_PREFIX =
"org.apache.iotdb.db.conf.directories.strategy.";
@@ -245,6 +246,16 @@ public class IoTDBConfig {
private String triggerDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TRIGGER_FOLDER_NAME;
+ /** External lib directory for properties loader, stores user-uploaded JAR files */
+ private String externalPropertiesLoaderDir =
+ IoTDBConstant.EXT_FOLDER_NAME
+ + File.separator
+ + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME;
+
+ /** External lib directory for limiter, stores user uploaded JAR files */
+ private String externalLimiterDir =
+ IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER;
+
/** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */
private String[] dataDirs = {"data" + File.separator + "data"};
@@ -988,6 +999,8 @@ public class IoTDBConfig {
udfDir = addHomeDir(udfDir);
triggerDir = addHomeDir(triggerDir);
operationSyncLogDir = addHomeDir(operationSyncLogDir);
+ externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir);
+ externalLimiterDir = addHomeDir(externalLimiterDir);
if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) {
String hdfsDir = getHdfsDir();
@@ -1212,6 +1225,22 @@ public class IoTDBConfig {
this.triggerDir = triggerDir;
}
+ public String getExternalPropertiesLoaderDir() {
+ return externalPropertiesLoaderDir;
+ }
+
+ public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) {
+ this.externalPropertiesLoaderDir = externalPropertiesLoaderDir;
+ }
+
+ public String getExternalLimiterDir() {
+ return externalLimiterDir;
+ }
+
+ public void setExternalLimiterDir(String externalLimiterDir) {
+ this.externalLimiterDir = externalLimiterDir;
+ }
+
public String getMultiDirStrategyClassName() {
return multiDirStrategyClassName;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 1294002707..337a9e0a6e 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -191,6 +191,10 @@ public class IoTDBConstant {
// compaction mods of previous version (<0.13)
public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods";
+ public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader";
+
+ public static final String EXT_LIMITER = "limiter";
+
// client version number
public enum ClientVersion {
V_0_12,
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 19d57c8699..cdb0057a2d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -45,8 +45,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.MalformedURLException;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Properties;
public class IoTDBDescriptor {
@@ -114,6 +117,52 @@ public class IoTDBDescriptor {
}
}
+ /**
+ * get external props url location
+ *
+ * @return url object if location exit, otherwise null.
+ */
+ public Path getExternalPropsPath() {
+ // Check if a config-directory was specified first.
+ String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
+ // If it wasn't, check if a home directory was provided (This usually contains a config)
+ if (urlString == null) {
+ urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
+ if (urlString != null) {
+ urlString =
+ urlString
+ + File.separatorChar
+ + "conf"
+ + File.separatorChar
+ + IoTDBConfig.EXTERNAL_CONFIG_NAME;
+ } else {
+ // If this too wasn't provided, try to find a default config in the root of the classpath.
+ URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME);
+ if (uri != null) {
+ try {
+ return Paths.get(uri.toURI());
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+ logger.warn(
+ "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading "
+ + "config file {}, use default configuration",
+ IoTDBConfig.EXTERNAL_CONFIG_NAME);
+ // update all data seriesPath
+ conf.updatePath();
+ return null;
+ }
+ }
+ // If a config location was provided, but it doesn't end with a properties file,
+ // append the default location.
+ else if (!urlString.endsWith(".properties")) {
+ urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME);
+ }
+
+ return Paths.get(urlString);
+ }
+
/** load an property file and set TsfileDBConfig variables. */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private void loadProps() {
@@ -129,814 +178,817 @@ public class IoTDBDescriptor {
Properties properties = new Properties();
properties.load(inputStream);
- conf.setRpcAddress(properties.getProperty("rpc_address", conf.getRpcAddress()));
- replaceHostnameWithIP();
+ loadProperties(properties);
- conf.setRpcThriftCompressionEnable(
- Boolean.parseBoolean(
- properties.getProperty(
- "rpc_thrift_compression_enable",
- Boolean.toString(conf.isRpcThriftCompressionEnable()))));
+ } catch (FileNotFoundException e) {
+ logger.warn("Fail to find config file {}", url, e);
+ } catch (IOException e) {
+ logger.warn("Cannot load config file, use default configuration", e);
+ } catch (Exception e) {
+ logger.warn("Incorrect format in config file, use default configuration", e);
+ } finally {
+ // update all data seriesPath
+ conf.updatePath();
+ // update instance in metric
+ MetricConfigDescriptor.getInstance()
+ .getMetricConfig()
+ .updateRpcInstance(conf.getRpcAddress(), conf.getRpcPort());
+ }
+ }
- conf.setRpcAdvancedCompressionEnable(
- Boolean.parseBoolean(
- properties.getProperty(
- "rpc_advanced_compression_enable",
- Boolean.toString(conf.isRpcAdvancedCompressionEnable()))));
+ public void loadProperties(Properties properties) throws UnknownHostException {
- conf.setRpcPort(
- Integer.parseInt(
- properties.getProperty("rpc_port", Integer.toString(conf.getRpcPort()))));
+ conf.setRpcAddress(properties.getProperty("rpc_address", conf.getRpcAddress()));
+ replaceHostnameWithIP();
- conf.setEnableInfluxDBRpcService(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_influxdb_rpc_service",
- Boolean.toString(conf.isEnableInfluxDBRpcService()))));
+ conf.setRpcThriftCompressionEnable(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "rpc_thrift_compression_enable",
+ Boolean.toString(conf.isRpcThriftCompressionEnable()))));
- conf.setInfluxDBRpcPort(
- Integer.parseInt(
- properties.getProperty(
- "influxdb_rpc_port", Integer.toString(conf.getInfluxDBRpcPort()))));
+ conf.setRpcAdvancedCompressionEnable(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "rpc_advanced_compression_enable",
+ Boolean.toString(conf.isRpcAdvancedCompressionEnable()))));
- conf.setTimestampPrecision(
- properties.getProperty("timestamp_precision", conf.getTimestampPrecision()));
+ conf.setRpcPort(
+ Integer.parseInt(properties.getProperty("rpc_port", Integer.toString(conf.getRpcPort()))));
- conf.setBufferedArraysMemoryProportion(
- Double.parseDouble(
- properties.getProperty(
- "buffered_arrays_memory_proportion",
- Double.toString(conf.getBufferedArraysMemoryProportion()))));
+ conf.setEnableInfluxDBRpcService(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_influxdb_rpc_service",
+ Boolean.toString(conf.isEnableInfluxDBRpcService()))));
- conf.setTimeIndexMemoryProportion(
- Double.parseDouble(
- properties.getProperty(
- "time_index_memory_proportion",
- Double.toString(conf.getTimeIndexMemoryProportion()))));
+ conf.setInfluxDBRpcPort(
+ Integer.parseInt(
+ properties.getProperty(
+ "influxdb_rpc_port", Integer.toString(conf.getInfluxDBRpcPort()))));
- conf.setFlushProportion(
- Double.parseDouble(
- properties.getProperty(
- "flush_proportion", Double.toString(conf.getFlushProportion()))));
+ conf.setTimestampPrecision(
+ properties.getProperty("timestamp_precision", conf.getTimestampPrecision()));
- conf.setRejectProportion(
- Double.parseDouble(
- properties.getProperty(
- "reject_proportion", Double.toString(conf.getRejectProportion()))));
+ conf.setBufferedArraysMemoryProportion(
+ Double.parseDouble(
+ properties.getProperty(
+ "buffered_arrays_memory_proportion",
+ Double.toString(conf.getBufferedArraysMemoryProportion()))));
- conf.setStorageGroupSizeReportThreshold(
- Long.parseLong(
- properties.getProperty(
- "storage_group_report_threshold",
- Long.toString(conf.getStorageGroupSizeReportThreshold()))));
+ conf.setTimeIndexMemoryProportion(
+ Double.parseDouble(
+ properties.getProperty(
+ "time_index_memory_proportion",
+ Double.toString(conf.getTimeIndexMemoryProportion()))));
- conf.setMetaDataCacheEnable(
- Boolean.parseBoolean(
- properties.getProperty(
- "meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable()))));
+ conf.setFlushProportion(
+ Double.parseDouble(
+ properties.getProperty(
+ "flush_proportion", Double.toString(conf.getFlushProportion()))));
- conf.setEnableLastCache(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_last_cache", Boolean.toString(conf.isLastCacheEnabled()))));
+ conf.setRejectProportion(
+ Double.parseDouble(
+ properties.getProperty(
+ "reject_proportion", Double.toString(conf.getRejectProportion()))));
+
+ conf.setStorageGroupSizeReportThreshold(
+ Long.parseLong(
+ properties.getProperty(
+ "storage_group_report_threshold",
+ Long.toString(conf.getStorageGroupSizeReportThreshold()))));
- initMemoryAllocate(properties);
+ conf.setMetaDataCacheEnable(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "meta_data_cache_enable", Boolean.toString(conf.isMetaDataCacheEnable()))));
- loadWALProps(properties);
+ conf.setEnableLastCache(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_last_cache", Boolean.toString(conf.isLastCacheEnabled()))));
- String systemDir = properties.getProperty("system_dir");
- if (systemDir == null) {
- systemDir = properties.getProperty("base_dir");
- if (systemDir != null) {
- systemDir = FilePathUtils.regularizePath(systemDir) + IoTDBConstant.SYSTEM_FOLDER_NAME;
- } else {
- systemDir = conf.getSystemDir();
- }
+ initMemoryAllocate(properties);
+
+ loadWALProps(properties);
+
+ String systemDir = properties.getProperty("system_dir");
+ if (systemDir == null) {
+ systemDir = properties.getProperty("base_dir");
+ if (systemDir != null) {
+ systemDir = FilePathUtils.regularizePath(systemDir) + IoTDBConstant.SYSTEM_FOLDER_NAME;
+ } else {
+ systemDir = conf.getSystemDir();
}
- conf.setSystemDir(systemDir);
+ }
+ conf.setSystemDir(systemDir);
- conf.setSchemaDir(
- FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SCHEMA_FOLDER_NAME);
+ conf.setSchemaDir(
+ FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SCHEMA_FOLDER_NAME);
- conf.setSyncDir(
- FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SYNC_FOLDER_NAME);
+ conf.setSyncDir(
+ FilePathUtils.regularizePath(conf.getSystemDir()) + IoTDBConstant.SYNC_FOLDER_NAME);
- conf.setQueryDir(
- FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME));
+ conf.setQueryDir(
+ FilePathUtils.regularizePath(conf.getSystemDir() + IoTDBConstant.QUERY_FOLDER_NAME));
- conf.setTracingDir(properties.getProperty("tracing_dir", conf.getTracingDir()));
+ conf.setTracingDir(properties.getProperty("tracing_dir", conf.getTracingDir()));
- conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0]).split(","));
+ conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0]).split(","));
- conf.setWalDir(properties.getProperty("wal_dir", conf.getWalDir()));
+ conf.setWalDir(properties.getProperty("wal_dir", conf.getWalDir()));
- int mlogBufferSize =
- Integer.parseInt(
- properties.getProperty(
- "mlog_buffer_size", Integer.toString(conf.getMlogBufferSize())));
- if (mlogBufferSize > 0) {
- conf.setMlogBufferSize(mlogBufferSize);
- }
+ int mlogBufferSize =
+ Integer.parseInt(
+ properties.getProperty("mlog_buffer_size", Integer.toString(conf.getMlogBufferSize())));
+ if (mlogBufferSize > 0) {
+ conf.setMlogBufferSize(mlogBufferSize);
+ }
- long forceMlogPeriodInMs =
- Long.parseLong(
- properties.getProperty(
- "sync_mlog_period_in_ms", Long.toString(conf.getSyncMlogPeriodInMs())));
- if (forceMlogPeriodInMs > 0) {
- conf.setSyncMlogPeriodInMs(forceMlogPeriodInMs);
- }
+ long forceMlogPeriodInMs =
+ Long.parseLong(
+ properties.getProperty(
+ "sync_mlog_period_in_ms", Long.toString(conf.getSyncMlogPeriodInMs())));
+ if (forceMlogPeriodInMs > 0) {
+ conf.setSyncMlogPeriodInMs(forceMlogPeriodInMs);
+ }
- conf.setMultiDirStrategyClassName(
- properties.getProperty("multi_dir_strategy", conf.getMultiDirStrategyClassName()));
+ conf.setMultiDirStrategyClassName(
+ properties.getProperty("multi_dir_strategy", conf.getMultiDirStrategyClassName()));
- conf.setBatchSize(
- Integer.parseInt(
- properties.getProperty("batch_size", Integer.toString(conf.getBatchSize()))));
+ conf.setBatchSize(
+ Integer.parseInt(
+ properties.getProperty("batch_size", Integer.toString(conf.getBatchSize()))));
- conf.setEnableMemControl(
- (Boolean.parseBoolean(
- properties.getProperty(
- "enable_mem_control", Boolean.toString(conf.isEnableMemControl())))));
- logger.info("IoTDB enable memory control: {}", conf.isEnableMemControl());
+ conf.setEnableMemControl(
+ (Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_mem_control", Boolean.toString(conf.isEnableMemControl())))));
+ logger.info("IoTDB enable memory control: {}", conf.isEnableMemControl());
- long seqTsFileSize =
- Long.parseLong(
- properties
- .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize()))
- .trim());
- if (seqTsFileSize >= 0) {
- conf.setSeqTsFileSize(seqTsFileSize);
- }
+ long seqTsFileSize =
+ Long.parseLong(
+ properties
+ .getProperty("seq_tsfile_size", Long.toString(conf.getSeqTsFileSize()))
+ .trim());
+ if (seqTsFileSize >= 0) {
+ conf.setSeqTsFileSize(seqTsFileSize);
+ }
- long unSeqTsFileSize =
- Long.parseLong(
- properties
- .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize()))
- .trim());
- if (unSeqTsFileSize >= 0) {
- conf.setUnSeqTsFileSize(unSeqTsFileSize);
- }
+ long unSeqTsFileSize =
+ Long.parseLong(
+ properties
+ .getProperty("unseq_tsfile_size", Long.toString(conf.getUnSeqTsFileSize()))
+ .trim());
+ if (unSeqTsFileSize >= 0) {
+ conf.setUnSeqTsFileSize(unSeqTsFileSize);
+ }
- long memTableSizeThreshold =
- Long.parseLong(
- properties
- .getProperty(
- "memtable_size_threshold", Long.toString(conf.getMemtableSizeThreshold()))
- .trim());
- if (memTableSizeThreshold > 0) {
- conf.setMemtableSizeThreshold(memTableSizeThreshold);
- }
+ long memTableSizeThreshold =
+ Long.parseLong(
+ properties
+ .getProperty(
+ "memtable_size_threshold", Long.toString(conf.getMemtableSizeThreshold()))
+ .trim());
+ if (memTableSizeThreshold > 0) {
+ conf.setMemtableSizeThreshold(memTableSizeThreshold);
+ }
- conf.setAvgSeriesPointNumberThreshold(
- Integer.parseInt(
- properties.getProperty(
- "avg_series_point_number_threshold",
- Integer.toString(conf.getAvgSeriesPointNumberThreshold()))));
+ conf.setAvgSeriesPointNumberThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "avg_series_point_number_threshold",
+ Integer.toString(conf.getAvgSeriesPointNumberThreshold()))));
- conf.setCheckPeriodWhenInsertBlocked(
- Integer.parseInt(
- properties.getProperty(
- "check_period_when_insert_blocked",
- Integer.toString(conf.getCheckPeriodWhenInsertBlocked()))));
+ conf.setCheckPeriodWhenInsertBlocked(
+ Integer.parseInt(
+ properties.getProperty(
+ "check_period_when_insert_blocked",
+ Integer.toString(conf.getCheckPeriodWhenInsertBlocked()))));
- conf.setMaxWaitingTimeWhenInsertBlocked(
- Integer.parseInt(
- properties.getProperty(
- "max_waiting_time_when_insert_blocked",
- Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked()))));
+ conf.setMaxWaitingTimeWhenInsertBlocked(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_waiting_time_when_insert_blocked",
+ Integer.toString(conf.getMaxWaitingTimeWhenInsertBlocked()))));
- conf.setEstimatedSeriesSize(
- Integer.parseInt(
- properties.getProperty(
- "estimated_series_size", Integer.toString(conf.getEstimatedSeriesSize()))));
+ conf.setEstimatedSeriesSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "estimated_series_size", Integer.toString(conf.getEstimatedSeriesSize()))));
- conf.setIoTaskQueueSizeForFlushing(
- Integer.parseInt(
- properties.getProperty(
- "io_task_queue_size_for_flushing",
- Integer.toString(conf.getIoTaskQueueSizeForFlushing()))));
+ conf.setIoTaskQueueSizeForFlushing(
+ Integer.parseInt(
+ properties.getProperty(
+ "io_task_queue_size_for_flushing",
+ Integer.toString(conf.getIoTaskQueueSizeForFlushing()))));
- conf.setCompactionScheduleIntervalInMs(
- Long.parseLong(
- properties.getProperty(
- "compaction_schedule_interval_in_ms",
- Long.toString(conf.getCompactionScheduleIntervalInMs()))));
+ conf.setCompactionScheduleIntervalInMs(
+ Long.parseLong(
+ properties.getProperty(
+ "compaction_schedule_interval_in_ms",
+ Long.toString(conf.getCompactionScheduleIntervalInMs()))));
- conf.setCompactionSubmissionIntervalInMs(
- Long.parseLong(
- properties.getProperty(
- "compaction_submission_interval_in_ms",
- Long.toString(conf.getCompactionSubmissionIntervalInMs()))));
+ conf.setCompactionSubmissionIntervalInMs(
+ Long.parseLong(
+ properties.getProperty(
+ "compaction_submission_interval_in_ms",
+ Long.toString(conf.getCompactionSubmissionIntervalInMs()))));
- conf.setEnableCrossSpaceCompaction(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_cross_space_compaction",
- Boolean.toString(conf.isEnableCrossSpaceCompaction()))));
+ conf.setEnableCrossSpaceCompaction(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_cross_space_compaction",
+ Boolean.toString(conf.isEnableCrossSpaceCompaction()))));
- conf.setEnableSeqSpaceCompaction(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_seq_space_compaction",
- Boolean.toString(conf.isEnableSeqSpaceCompaction()))));
+ conf.setEnableSeqSpaceCompaction(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_seq_space_compaction",
+ Boolean.toString(conf.isEnableSeqSpaceCompaction()))));
- conf.setEnableUnseqSpaceCompaction(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_unseq_space_compaction",
- Boolean.toString(conf.isEnableUnseqSpaceCompaction()))));
+ conf.setEnableUnseqSpaceCompaction(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_unseq_space_compaction",
+ Boolean.toString(conf.isEnableUnseqSpaceCompaction()))));
- conf.setCrossCompactionStrategy(
- CrossCompactionStrategy.getCrossCompactionStrategy(
- properties.getProperty(
- "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString())));
+ conf.setCrossCompactionStrategy(
+ CrossCompactionStrategy.getCrossCompactionStrategy(
+ properties.getProperty(
+ "cross_compaction_strategy", conf.getCrossCompactionStrategy().toString())));
- conf.setInnerCompactionStrategy(
- InnerCompactionStrategy.getInnerCompactionStrategy(
- properties.getProperty(
- "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString())));
+ conf.setInnerCompactionStrategy(
+ InnerCompactionStrategy.getInnerCompactionStrategy(
+ properties.getProperty(
+ "inner_compaction_strategy", conf.getInnerCompactionStrategy().toString())));
- conf.setCompactionPriority(
- CompactionPriority.valueOf(
- properties.getProperty(
- "compaction_priority", conf.getCompactionPriority().toString())));
+ conf.setCompactionPriority(
+ CompactionPriority.valueOf(
+ properties.getProperty(
+ "compaction_priority", conf.getCompactionPriority().toString())));
- int subtaskNum =
- Integer.parseInt(
- properties.getProperty(
- "sub_compaction_thread_num", Integer.toString(conf.getSubCompactionTaskNum())));
- subtaskNum = subtaskNum <= 0 ? 1 : subtaskNum;
- conf.setSubCompactionTaskNum(subtaskNum);
+ int subtaskNum =
+ Integer.parseInt(
+ properties.getProperty(
+ "sub_compaction_thread_num", Integer.toString(conf.getSubCompactionTaskNum())));
+ subtaskNum = subtaskNum <= 0 ? 1 : subtaskNum;
+ conf.setSubCompactionTaskNum(subtaskNum);
- conf.setQueryTimeoutThreshold(
- Integer.parseInt(
- properties.getProperty(
- "query_timeout_threshold", Integer.toString(conf.getQueryTimeoutThreshold()))));
+ conf.setQueryTimeoutThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "query_timeout_threshold", Integer.toString(conf.getQueryTimeoutThreshold()))));
- conf.setSessionTimeoutThreshold(
- Integer.parseInt(
- properties.getProperty(
- "session_timeout_threshold",
- Integer.toString(conf.getSessionTimeoutThreshold()))));
+ conf.setSessionTimeoutThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "session_timeout_threshold", Integer.toString(conf.getSessionTimeoutThreshold()))));
- conf.setSyncEnable(
- Boolean.parseBoolean(
- properties.getProperty("is_sync_enable", Boolean.toString(conf.isSyncEnable()))));
+ conf.setSyncEnable(
+ Boolean.parseBoolean(
+ properties.getProperty("is_sync_enable", Boolean.toString(conf.isSyncEnable()))));
- conf.setSyncServerPort(
- Integer.parseInt(
- properties
- .getProperty("sync_server_port", Integer.toString(conf.getSyncServerPort()))
- .trim()));
+ conf.setSyncServerPort(
+ Integer.parseInt(
+ properties
+ .getProperty("sync_server_port", Integer.toString(conf.getSyncServerPort()))
+ .trim()));
- conf.setIpWhiteList(properties.getProperty("ip_white_list", conf.getIpWhiteList()));
+ conf.setIpWhiteList(properties.getProperty("ip_white_list", conf.getIpWhiteList()));
- conf.setConcurrentFlushThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_flush_thread", Integer.toString(conf.getConcurrentFlushThread()))));
+ conf.setConcurrentFlushThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_flush_thread", Integer.toString(conf.getConcurrentFlushThread()))));
- if (conf.getConcurrentFlushThread() <= 0) {
- conf.setConcurrentFlushThread(Runtime.getRuntime().availableProcessors());
- }
+ if (conf.getConcurrentFlushThread() <= 0) {
+ conf.setConcurrentFlushThread(Runtime.getRuntime().availableProcessors());
+ }
- // start: index parameter setting
- conf.setIndexRootFolder(properties.getProperty("index_root_dir", conf.getIndexRootFolder()));
+ // start: index parameter setting
+ conf.setIndexRootFolder(properties.getProperty("index_root_dir", conf.getIndexRootFolder()));
- conf.setEnableIndex(
- Boolean.parseBoolean(
- properties.getProperty("enable_index", Boolean.toString(conf.isEnableIndex()))));
+ conf.setEnableIndex(
+ Boolean.parseBoolean(
+ properties.getProperty("enable_index", Boolean.toString(conf.isEnableIndex()))));
- conf.setConcurrentIndexBuildThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_index_build_thread",
- Integer.toString(conf.getConcurrentIndexBuildThread()))));
- if (conf.getConcurrentIndexBuildThread() <= 0) {
- conf.setConcurrentIndexBuildThread(Runtime.getRuntime().availableProcessors());
- }
+ conf.setConcurrentIndexBuildThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_index_build_thread",
+ Integer.toString(conf.getConcurrentIndexBuildThread()))));
+ if (conf.getConcurrentIndexBuildThread() <= 0) {
+ conf.setConcurrentIndexBuildThread(Runtime.getRuntime().availableProcessors());
+ }
- conf.setDefaultIndexWindowRange(
- Integer.parseInt(
- properties.getProperty(
- "default_index_window_range",
- Integer.toString(conf.getDefaultIndexWindowRange()))));
+ conf.setDefaultIndexWindowRange(
+ Integer.parseInt(
+ properties.getProperty(
+ "default_index_window_range",
+ Integer.toString(conf.getDefaultIndexWindowRange()))));
- conf.setIndexBufferSize(
- Long.parseLong(
- properties.getProperty(
- "index_buffer_size", Long.toString(conf.getIndexBufferSize()))));
- // end: index parameter setting
+ conf.setIndexBufferSize(
+ Long.parseLong(
+ properties.getProperty("index_buffer_size", Long.toString(conf.getIndexBufferSize()))));
+ // end: index parameter setting
- conf.setConcurrentQueryThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_query_thread", Integer.toString(conf.getConcurrentQueryThread()))));
+ conf.setConcurrentQueryThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_query_thread", Integer.toString(conf.getConcurrentQueryThread()))));
- if (conf.getConcurrentQueryThread() <= 0) {
- conf.setConcurrentQueryThread(Runtime.getRuntime().availableProcessors());
- }
+ if (conf.getConcurrentQueryThread() <= 0) {
+ conf.setConcurrentQueryThread(Runtime.getRuntime().availableProcessors());
+ }
- conf.setConcurrentSubRawQueryThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_sub_rawQuery_thread",
- Integer.toString(conf.getConcurrentSubRawQueryThread()))));
+ conf.setConcurrentSubRawQueryThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_sub_rawQuery_thread",
+ Integer.toString(conf.getConcurrentSubRawQueryThread()))));
- if (conf.getConcurrentSubRawQueryThread() <= 0) {
- conf.setConcurrentSubRawQueryThread(Runtime.getRuntime().availableProcessors());
- }
+ if (conf.getConcurrentSubRawQueryThread() <= 0) {
+ conf.setConcurrentSubRawQueryThread(Runtime.getRuntime().availableProcessors());
+ }
- conf.setRawQueryBlockingQueueCapacity(
- Integer.parseInt(
- properties.getProperty(
- "raw_query_blocking_queue_capacity",
- Integer.toString(conf.getRawQueryBlockingQueueCapacity()))));
+ conf.setRawQueryBlockingQueueCapacity(
+ Integer.parseInt(
+ properties.getProperty(
+ "raw_query_blocking_queue_capacity",
+ Integer.toString(conf.getRawQueryBlockingQueueCapacity()))));
- conf.setmManagerCacheSize(
- Integer.parseInt(
- properties
- .getProperty(
- "metadata_node_cache_size", Integer.toString(conf.getmManagerCacheSize()))
- .trim()));
+ conf.setmManagerCacheSize(
+ Integer.parseInt(
+ properties
+ .getProperty(
+ "metadata_node_cache_size", Integer.toString(conf.getmManagerCacheSize()))
+ .trim()));
- conf.setmRemoteSchemaCacheSize(
- Integer.parseInt(
- properties
- .getProperty(
- "remote_schema_cache_size",
- Integer.toString(conf.getmRemoteSchemaCacheSize()))
- .trim()));
+ conf.setmRemoteSchemaCacheSize(
+ Integer.parseInt(
+ properties
+ .getProperty(
+ "remote_schema_cache_size", Integer.toString(conf.getmRemoteSchemaCacheSize()))
+ .trim()));
- conf.setLanguageVersion(
- properties.getProperty("language_version", conf.getLanguageVersion()).trim());
+ conf.setLanguageVersion(
+ properties.getProperty("language_version", conf.getLanguageVersion()).trim());
- if (properties.containsKey("chunk_buffer_pool_enable")) {
- conf.setChunkBufferPoolEnable(
- Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable")));
- }
+ if (properties.containsKey("chunk_buffer_pool_enable")) {
+ conf.setChunkBufferPoolEnable(
+ Boolean.parseBoolean(properties.getProperty("chunk_buffer_pool_enable")));
+ }
- conf.setEnableExternalSort(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_external_sort", Boolean.toString(conf.isEnableExternalSort()))));
- conf.setExternalSortThreshold(
- Integer.parseInt(
- properties.getProperty(
- "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold()))));
- conf.setUpgradeThreadNum(
- Integer.parseInt(
- properties.getProperty(
- "upgrade_thread_num", Integer.toString(conf.getUpgradeThreadNum()))));
- conf.setCrossCompactionMemoryBudget(
- Long.parseLong(
- properties.getProperty(
- "cross_compaction_memory_budget",
- Long.toString(conf.getCrossCompactionMemoryBudget()))));
- conf.setCrossCompactionFileSelectionTimeBudget(
- Long.parseLong(
- properties.getProperty(
- "cross_compaction_file_selection_time_budget",
- Long.toString(conf.getCrossCompactionFileSelectionTimeBudget()))));
- conf.setMergeIntervalSec(
- Long.parseLong(
- properties.getProperty(
- "merge_interval_sec", Long.toString(conf.getMergeIntervalSec()))));
- conf.setConcurrentCompactionThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_compaction_thread",
- Integer.toString(conf.getConcurrentCompactionThread()))));
- conf.setTargetCompactionFileSize(
- Long.parseLong(
- properties.getProperty(
- "target_compaction_file_size",
- Long.toString(conf.getTargetCompactionFileSize()))));
- conf.setTargetChunkSize(
- Long.parseLong(
- properties.getProperty(
- "target_chunk_size", Long.toString(conf.getTargetChunkSize()))));
- conf.setTargetChunkPointNum(
- Long.parseLong(
- properties.getProperty(
- "target_chunk_point_num", Long.toString(conf.getTargetChunkPointNum()))));
- conf.setChunkPointNumLowerBoundInCompaction(
- Long.parseLong(
- properties.getProperty(
- "chunk_size_lower_bound_in_compaction",
- Long.toString(conf.getChunkPointNumLowerBoundInCompaction()))));
- conf.setChunkSizeLowerBoundInCompaction(
- Long.parseLong(
- properties.getProperty(
- "chunk_size_lower_bound_in_compaction",
- Long.toString(conf.getChunkSizeLowerBoundInCompaction()))));
- conf.setMaxInnerCompactionCandidateFileNum(
- Integer.parseInt(
- properties.getProperty(
- "max_inner_compaction_candidate_file_num",
- Integer.toString(conf.getMaxInnerCompactionCandidateFileNum()))));
- conf.setMaxCrossCompactionCandidateFileNum(
- Integer.parseInt(
- properties.getProperty(
- "max_cross_compaction_candidate_file_num",
- Integer.toString(conf.getMaxCrossCompactionCandidateFileNum()))));
+ conf.setEnableExternalSort(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_external_sort", Boolean.toString(conf.isEnableExternalSort()))));
+ conf.setExternalSortThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "external_sort_threshold", Integer.toString(conf.getExternalSortThreshold()))));
+ conf.setUpgradeThreadNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "upgrade_thread_num", Integer.toString(conf.getUpgradeThreadNum()))));
+ conf.setCrossCompactionMemoryBudget(
+ Long.parseLong(
+ properties.getProperty(
+ "cross_compaction_memory_budget",
+ Long.toString(conf.getCrossCompactionMemoryBudget()))));
+ conf.setCrossCompactionFileSelectionTimeBudget(
+ Long.parseLong(
+ properties.getProperty(
+ "cross_compaction_file_selection_time_budget",
+ Long.toString(conf.getCrossCompactionFileSelectionTimeBudget()))));
+ conf.setMergeIntervalSec(
+ Long.parseLong(
+ properties.getProperty(
+ "merge_interval_sec", Long.toString(conf.getMergeIntervalSec()))));
+ conf.setConcurrentCompactionThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_compaction_thread",
+ Integer.toString(conf.getConcurrentCompactionThread()))));
+ conf.setTargetCompactionFileSize(
+ Long.parseLong(
+ properties.getProperty(
+ "target_compaction_file_size", Long.toString(conf.getTargetCompactionFileSize()))));
+ conf.setTargetChunkSize(
+ Long.parseLong(
+ properties.getProperty("target_chunk_size", Long.toString(conf.getTargetChunkSize()))));
+ conf.setTargetChunkPointNum(
+ Long.parseLong(
+ properties.getProperty(
+ "target_chunk_point_num", Long.toString(conf.getTargetChunkPointNum()))));
+ conf.setChunkPointNumLowerBoundInCompaction(
+ Long.parseLong(
+ properties.getProperty(
+ "chunk_size_lower_bound_in_compaction",
+ Long.toString(conf.getChunkPointNumLowerBoundInCompaction()))));
+ conf.setChunkSizeLowerBoundInCompaction(
+ Long.parseLong(
+ properties.getProperty(
+ "chunk_size_lower_bound_in_compaction",
+ Long.toString(conf.getChunkSizeLowerBoundInCompaction()))));
+ conf.setMaxInnerCompactionCandidateFileNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_inner_compaction_candidate_file_num",
+ Integer.toString(conf.getMaxInnerCompactionCandidateFileNum()))));
+ conf.setMaxCrossCompactionCandidateFileNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_cross_compaction_candidate_file_num",
+ Integer.toString(conf.getMaxCrossCompactionCandidateFileNum()))));
- conf.setCompactionWriteThroughputMbPerSec(
- Integer.parseInt(
- properties.getProperty(
- "compaction_write_throughput_mb_per_sec",
- Integer.toString(conf.getCompactionWriteThroughputMbPerSec()))));
+ conf.setCompactionWriteThroughputMbPerSec(
+ Integer.parseInt(
+ properties.getProperty(
+ "compaction_write_throughput_mb_per_sec",
+ Integer.toString(conf.getCompactionWriteThroughputMbPerSec()))));
- conf.setEnablePartialInsert(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert()))));
+ conf.setEnablePartialInsert(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_partial_insert", String.valueOf(conf.isEnablePartialInsert()))));
- conf.setEnableMTreeSnapshot(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_mtree_snapshot", Boolean.toString(conf.isEnableMTreeSnapshot()))));
- conf.setMtreeSnapshotInterval(
- Integer.parseInt(
- properties.getProperty(
- "mtree_snapshot_interval", Integer.toString(conf.getMtreeSnapshotInterval()))));
- conf.setMtreeSnapshotThresholdTime(
- Integer.parseInt(
- properties.getProperty(
- "mtree_snapshot_threshold_time",
- Integer.toString(conf.getMtreeSnapshotThresholdTime()))));
+ conf.setEnableMTreeSnapshot(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_mtree_snapshot", Boolean.toString(conf.isEnableMTreeSnapshot()))));
+ conf.setMtreeSnapshotInterval(
+ Integer.parseInt(
+ properties.getProperty(
+ "mtree_snapshot_interval", Integer.toString(conf.getMtreeSnapshotInterval()))));
+ conf.setMtreeSnapshotThresholdTime(
+ Integer.parseInt(
+ properties.getProperty(
+ "mtree_snapshot_threshold_time",
+ Integer.toString(conf.getMtreeSnapshotThresholdTime()))));
- int maxConcurrentClientNum =
- Integer.parseInt(
- properties.getProperty(
- "rpc_max_concurrent_client_num",
- Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim()));
- if (maxConcurrentClientNum <= 0) {
- maxConcurrentClientNum = 65535;
- }
+ int maxConcurrentClientNum =
+ Integer.parseInt(
+ properties.getProperty(
+ "rpc_max_concurrent_client_num",
+ Integer.toString(conf.getRpcMaxConcurrentClientNum()).trim()));
+ if (maxConcurrentClientNum <= 0) {
+ maxConcurrentClientNum = 65535;
+ }
- conf.setEnableWatermark(
- Boolean.parseBoolean(
- properties.getProperty(
- "watermark_module_opened", Boolean.toString(conf.isEnableWatermark()).trim())));
- conf.setWatermarkSecretKey(
- properties.getProperty("watermark_secret_key", conf.getWatermarkSecretKey()));
- conf.setWatermarkBitString(
- properties.getProperty("watermark_bit_string", conf.getWatermarkBitString()));
- conf.setWatermarkMethod(
- properties.getProperty("watermark_method", conf.getWatermarkMethod()));
+ conf.setEnableWatermark(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "watermark_module_opened", Boolean.toString(conf.isEnableWatermark()).trim())));
+ conf.setWatermarkSecretKey(
+ properties.getProperty("watermark_secret_key", conf.getWatermarkSecretKey()));
+ conf.setWatermarkBitString(
+ properties.getProperty("watermark_bit_string", conf.getWatermarkBitString()));
+ conf.setWatermarkMethod(properties.getProperty("watermark_method", conf.getWatermarkMethod()));
+
+ loadAutoCreateSchemaProps(properties);
+
+ conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum);
+
+ conf.setTsFileStorageFs(
+ properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().toString()));
+ conf.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
+ conf.setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
+ conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
+ conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
+ conf.setDfsNameServices(properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
+ conf.setDfsHaNamenodes(
+ properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
+ conf.setDfsHaAutomaticFailoverEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "dfs_ha_automatic_failover_enabled",
+ String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
+ conf.setDfsClientFailoverProxyProvider(
+ properties.getProperty(
+ "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider()));
+ conf.setUseKerberos(
+ Boolean.parseBoolean(
+ properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
+ conf.setKerberosKeytabFilePath(
+ properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath()));
+ conf.setKerberosPrincipal(
+ properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
- loadAutoCreateSchemaProps(properties);
+ conf.setDefaultTTL(
+ Long.parseLong(
+ properties.getProperty("default_ttl", String.valueOf(conf.getDefaultTTL()))));
- conf.setRpcMaxConcurrentClientNum(maxConcurrentClientNum);
-
- conf.setTsFileStorageFs(
- properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().toString()));
- conf.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
- conf.setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
- conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
- conf.setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
- conf.setDfsNameServices(
- properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
- conf.setDfsHaNamenodes(
- properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
- conf.setDfsHaAutomaticFailoverEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "dfs_ha_automatic_failover_enabled",
- String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
- conf.setDfsClientFailoverProxyProvider(
- properties.getProperty(
- "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider()));
- conf.setUseKerberos(
- Boolean.parseBoolean(
- properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
- conf.setKerberosKeytabFilePath(
- properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath()));
- conf.setKerberosPrincipal(
- properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
-
- conf.setDefaultTTL(
- Long.parseLong(
- properties.getProperty("default_ttl", String.valueOf(conf.getDefaultTTL()))));
+ conf.setAllowReadOnlyWhenErrorsOccur(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "allow_read_only_when_errors_occur",
+ String.valueOf(conf.isAllowReadOnlyWhenErrorsOccur()))));
- conf.setAllowReadOnlyWhenErrorsOccur(
- Boolean.parseBoolean(
- properties.getProperty(
- "allow_read_only_when_errors_occur",
- String.valueOf(conf.isAllowReadOnlyWhenErrorsOccur()))));
+ // the num of memtables in each storage group
+ conf.setConcurrentWritingTimePartition(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_writing_time_partition",
+ String.valueOf(conf.getConcurrentWritingTimePartition()))));
- // the num of memtables in each storage group
- conf.setConcurrentWritingTimePartition(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_writing_time_partition",
- String.valueOf(conf.getConcurrentWritingTimePartition()))));
+ // the default fill interval in LinearFill and PreviousFill
+ conf.setDefaultFillInterval(
+ Integer.parseInt(
+ properties.getProperty(
+ "default_fill_interval", String.valueOf(conf.getDefaultFillInterval()))));
- // the default fill interval in LinearFill and PreviousFill
- conf.setDefaultFillInterval(
- Integer.parseInt(
- properties.getProperty(
- "default_fill_interval", String.valueOf(conf.getDefaultFillInterval()))));
+ conf.setTagAttributeTotalSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "tag_attribute_total_size", String.valueOf(conf.getTagAttributeTotalSize()))));
- conf.setTagAttributeTotalSize(
- Integer.parseInt(
- properties.getProperty(
- "tag_attribute_total_size", String.valueOf(conf.getTagAttributeTotalSize()))));
+ conf.setTagAttributeFlushInterval(
+ Integer.parseInt(
+ properties.getProperty(
+ "tag_attribute_flush_interval",
+ String.valueOf(conf.getTagAttributeFlushInterval()))));
- conf.setTagAttributeFlushInterval(
- Integer.parseInt(
- properties.getProperty(
- "tag_attribute_flush_interval",
- String.valueOf(conf.getTagAttributeFlushInterval()))));
+ conf.setPrimitiveArraySize(
+ (Integer.parseInt(
+ properties.getProperty(
+ "primitive_array_size", String.valueOf(conf.getPrimitiveArraySize())))));
- conf.setPrimitiveArraySize(
- (Integer.parseInt(
- properties.getProperty(
- "primitive_array_size", String.valueOf(conf.getPrimitiveArraySize())))));
+ conf.setThriftMaxFrameSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize()))));
- conf.setThriftMaxFrameSize(
- Integer.parseInt(
- properties.getProperty(
- "thrift_max_frame_size", String.valueOf(conf.getThriftMaxFrameSize()))));
+ if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) {
+ conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2);
+ }
- if (conf.getThriftMaxFrameSize() < IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2) {
- conf.setThriftMaxFrameSize(IoTDBConstant.LEFT_SIZE_IN_REQUEST * 2);
- }
+ conf.setThriftDefaultBufferSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "thrift_init_buffer_size", String.valueOf(conf.getThriftDefaultBufferSize()))));
- conf.setThriftDefaultBufferSize(
- Integer.parseInt(
- properties.getProperty(
- "thrift_init_buffer_size", String.valueOf(conf.getThriftDefaultBufferSize()))));
+ conf.setFrequencyIntervalInMinute(
+ Integer.parseInt(
+ properties.getProperty(
+ "frequency_interval_in_minute",
+ String.valueOf(conf.getFrequencyIntervalInMinute()))));
- conf.setFrequencyIntervalInMinute(
- Integer.parseInt(
- properties.getProperty(
- "frequency_interval_in_minute",
- String.valueOf(conf.getFrequencyIntervalInMinute()))));
+ conf.setSlowQueryThreshold(
+ Long.parseLong(
+ properties.getProperty(
+ "slow_query_threshold", String.valueOf(conf.getSlowQueryThreshold()))));
- conf.setSlowQueryThreshold(
- Long.parseLong(
- properties.getProperty(
- "slow_query_threshold", String.valueOf(conf.getSlowQueryThreshold()))));
+ conf.setVirtualStorageGroupNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "virtual_storage_group_num", String.valueOf(conf.getVirtualStorageGroupNum()))));
- conf.setVirtualStorageGroupNum(
- Integer.parseInt(
- properties.getProperty(
- "virtual_storage_group_num", String.valueOf(conf.getVirtualStorageGroupNum()))));
+ conf.setRecoveryLogIntervalInMs(
+ Long.parseLong(
+ properties.getProperty(
+ "recovery_log_interval_in_ms", String.valueOf(conf.getRecoveryLogIntervalInMs()))));
- conf.setRecoveryLogIntervalInMs(
- Long.parseLong(
- properties.getProperty(
- "recovery_log_interval_in_ms",
- String.valueOf(conf.getRecoveryLogIntervalInMs()))));
+ conf.setConcurrentWindowEvaluationThread(
+ Integer.parseInt(
+ properties.getProperty(
+ "concurrent_window_evaluation_thread",
+ Integer.toString(conf.getConcurrentWindowEvaluationThread()))));
+ if (conf.getConcurrentWindowEvaluationThread() <= 0) {
+ conf.setConcurrentWindowEvaluationThread(Runtime.getRuntime().availableProcessors());
+ }
- conf.setConcurrentWindowEvaluationThread(
- Integer.parseInt(
- properties.getProperty(
- "concurrent_window_evaluation_thread",
- Integer.toString(conf.getConcurrentWindowEvaluationThread()))));
- if (conf.getConcurrentWindowEvaluationThread() <= 0) {
- conf.setConcurrentWindowEvaluationThread(Runtime.getRuntime().availableProcessors());
- }
+ conf.setMaxPendingWindowEvaluationTasks(
+ Integer.parseInt(
+ properties.getProperty(
+ "max_pending_window_evaluation_tasks",
+ Integer.toString(conf.getMaxPendingWindowEvaluationTasks()))));
+ if (conf.getMaxPendingWindowEvaluationTasks() <= 0) {
+ conf.setMaxPendingWindowEvaluationTasks(64);
+ }
- conf.setMaxPendingWindowEvaluationTasks(
- Integer.parseInt(
- properties.getProperty(
- "max_pending_window_evaluation_tasks",
- Integer.toString(conf.getMaxPendingWindowEvaluationTasks()))));
- if (conf.getMaxPendingWindowEvaluationTasks() <= 0) {
- conf.setMaxPendingWindowEvaluationTasks(64);
- }
+ // id table related configuration
+ // conf.setDeviceIDTransformationMethod(
+ // properties.getProperty(
+ // "device_id_transformation_method", conf.getDeviceIDTransformationMethod()));
+
+ // conf.setEnableIDTable(
+ // Boolean.parseBoolean(
+ // properties.getProperty("enable_id_table",
+ // String.valueOf(conf.isEnableIDTable()))));
+
+ // conf.setEnableIDTableLogFile(
+ // Boolean.parseBoolean(
+ // properties.getProperty(
+ // "enable_id_table_log_file",
+ // String.valueOf(conf.isEnableIDTableLogFile()))));
+
+ // mqtt
+ if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) {
+ conf.setMqttHost(properties.getProperty(IoTDBConstant.MQTT_HOST_NAME));
+ }
+ if (properties.getProperty(IoTDBConstant.MQTT_PORT_NAME) != null) {
+ conf.setMqttPort(Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_PORT_NAME)));
+ }
+ if (properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME) != null) {
+ conf.setMqttHandlerPoolSize(
+ Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME)));
+ }
+ if (properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME) != null) {
+ conf.setMqttPayloadFormatter(
+ properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME));
+ }
+ if (properties.getProperty(IoTDBConstant.ENABLE_MQTT) != null) {
+ conf.setEnableMQTTService(
+ Boolean.parseBoolean(properties.getProperty(IoTDBConstant.ENABLE_MQTT)));
+ }
+ if (properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE) != null) {
+ conf.setMqttMaxMessageSize(
+ Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE)));
+ }
- // id table related configuration
- // conf.setDeviceIDTransformationMethod(
- // properties.getProperty(
- // "device_id_transformation_method", conf.getDeviceIDTransformationMethod()));
-
- // conf.setEnableIDTable(
- // Boolean.parseBoolean(
- // properties.getProperty("enable_id_table",
- // String.valueOf(conf.isEnableIDTable()))));
-
- // conf.setEnableIDTableLogFile(
- // Boolean.parseBoolean(
- // properties.getProperty(
- // "enable_id_table_log_file",
- // String.valueOf(conf.isEnableIDTableLogFile()))));
-
- // mqtt
- if (properties.getProperty(IoTDBConstant.MQTT_HOST_NAME) != null) {
- conf.setMqttHost(properties.getProperty(IoTDBConstant.MQTT_HOST_NAME));
- }
- if (properties.getProperty(IoTDBConstant.MQTT_PORT_NAME) != null) {
- conf.setMqttPort(Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_PORT_NAME)));
- }
- if (properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME) != null) {
- conf.setMqttHandlerPoolSize(
- Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_HANDLER_POOL_SIZE_NAME)));
- }
- if (properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME) != null) {
- conf.setMqttPayloadFormatter(
- properties.getProperty(IoTDBConstant.MQTT_PAYLOAD_FORMATTER_NAME));
- }
- if (properties.getProperty(IoTDBConstant.ENABLE_MQTT) != null) {
- conf.setEnableMQTTService(
- Boolean.parseBoolean(properties.getProperty(IoTDBConstant.ENABLE_MQTT)));
- }
- if (properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE) != null) {
- conf.setMqttMaxMessageSize(
- Integer.parseInt(properties.getProperty(IoTDBConstant.MQTT_MAX_MESSAGE_SIZE)));
- }
+ conf.setAuthorizerProvider(
+ properties.getProperty("authorizer_provider_class", conf.getAuthorizerProvider()));
+ // if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed.
+ conf.setOpenIdProviderUrl(properties.getProperty("openID_url", conf.getOpenIdProviderUrl()));
- conf.setAuthorizerProvider(
- properties.getProperty("authorizer_provider_class", conf.getAuthorizerProvider()));
- // if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed.
- conf.setOpenIdProviderUrl(properties.getProperty("openID_url", conf.getOpenIdProviderUrl()));
+ conf.setEnablePartition(
+ Boolean.parseBoolean(
+ properties.getProperty("enable_partition", String.valueOf(conf.isEnablePartition()))));
- conf.setEnablePartition(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_partition", String.valueOf(conf.isEnablePartition()))));
+ conf.setPartitionInterval(
+ Long.parseLong(
+ properties.getProperty(
+ "partition_interval", String.valueOf(conf.getPartitionInterval()))));
- conf.setPartitionInterval(
- Long.parseLong(
- properties.getProperty(
- "partition_interval", String.valueOf(conf.getPartitionInterval()))));
+ conf.setAdminName(properties.getProperty("admin_name", conf.getAdminName()));
- conf.setAdminName(properties.getProperty("admin_name", conf.getAdminName()));
+ conf.setAdminPassword(properties.getProperty("admin_password", conf.getAdminPassword()));
- conf.setAdminPassword(properties.getProperty("admin_password", conf.getAdminPassword()));
+ conf.setSelectIntoInsertTabletPlanRowLimit(
+ Integer.parseInt(
+ properties.getProperty(
+ "select_into_insert_tablet_plan_row_limit",
+ String.valueOf(conf.getSelectIntoInsertTabletPlanRowLimit()))));
- conf.setSelectIntoInsertTabletPlanRowLimit(
- Integer.parseInt(
- properties.getProperty(
- "select_into_insert_tablet_plan_row_limit",
- String.valueOf(conf.getSelectIntoInsertTabletPlanRowLimit()))));
+ conf.setInsertMultiTabletEnableMultithreadingColumnThreshold(
+ Integer.parseInt(
+ properties.getProperty(
+ "insert_multi_tablet_enable_multithreading_column_threshold",
+ String.valueOf(conf.getInsertMultiTabletEnableMultithreadingColumnThreshold()))));
- conf.setInsertMultiTabletEnableMultithreadingColumnThreshold(
- Integer.parseInt(
- properties.getProperty(
- "insert_multi_tablet_enable_multithreading_column_threshold",
- String.valueOf(conf.getInsertMultiTabletEnableMultithreadingColumnThreshold()))));
+ conf.setEncryptDecryptProvider(
+ properties.getProperty(
+ "iotdb_server_encrypt_decrypt_provider", conf.getEncryptDecryptProvider()));
- conf.setEncryptDecryptProvider(
- properties.getProperty(
- "iotdb_server_encrypt_decrypt_provider", conf.getEncryptDecryptProvider()));
+ conf.setEncryptDecryptProviderParameter(
+ properties.getProperty(
+ "iotdb_server_encrypt_decrypt_provider_parameter",
+ conf.getEncryptDecryptProviderParameter()));
- conf.setEncryptDecryptProviderParameter(
- properties.getProperty(
- "iotdb_server_encrypt_decrypt_provider_parameter",
- conf.getEncryptDecryptProviderParameter()));
+ // set OperationSync config
+ conf.setEnableOperationSync(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_operation_sync", String.valueOf(conf.isEnableOperationSync()))));
- // set OperationSync config
- conf.setEnableOperationSync(
- Boolean.parseBoolean(
- properties.getProperty(
- "enable_operation_sync", String.valueOf(conf.isEnableOperationSync()))));
+ conf.setSecondaryAddress(
+ properties.getProperty("secondary_address", conf.getSecondaryAddress()));
- conf.setSecondaryAddress(
- properties.getProperty("secondary_address", conf.getSecondaryAddress()));
+ conf.setSecondaryPort(
+ Integer.parseInt(
+ properties.getProperty("secondary_port", String.valueOf(conf.getSecondaryPort()))));
- conf.setSecondaryPort(
- Integer.parseInt(
- properties.getProperty("secondary_port", String.valueOf(conf.getSecondaryPort()))));
+ conf.setSecondaryUser(properties.getProperty("secondary_user", conf.getSecondaryUser()));
- conf.setSecondaryUser(properties.getProperty("secondary_user", conf.getSecondaryUser()));
+ conf.setSecondaryPassword(
+ properties.getProperty("secondary_password", conf.getSecondaryPassword()));
- conf.setSecondaryPassword(
- properties.getProperty("secondary_password", conf.getSecondaryPassword()));
+ conf.setOperationSyncSessionConcurrencySize(
+ Integer.parseInt(
+ properties.getProperty(
+ "operation_sync_session_concurrency_size",
+ String.valueOf(conf.getOperationSyncSessionConcurrencySize()))));
- conf.setOperationSyncSessionConcurrencySize(
- Integer.parseInt(
- properties.getProperty(
- "operation_sync_session_concurrency_size",
- String.valueOf(conf.getOperationSyncSessionConcurrencySize()))));
+ conf.setOperationSyncLogDir(
+ properties.getProperty("operation_sync_log_dir", conf.getOperationSyncLogDir()));
- conf.setOperationSyncLogDir(
- properties.getProperty("operation_sync_log_dir", conf.getOperationSyncLogDir()));
+ conf.setOperationSyncLogValidity(
+ Integer.parseInt(
+ properties.getProperty(
+ "operation_sync_log_file_validity",
+ String.valueOf(conf.getOperationSyncLogValidity()))));
- conf.setOperationSyncLogValidity(
- Integer.parseInt(
- properties.getProperty(
- "operation_sync_log_file_validity",
- String.valueOf(conf.getOperationSyncLogValidity()))));
+ conf.setOperationSyncLogNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "operation_sync_log_file_num", String.valueOf(conf.getOperationSyncLogNum()))));
- conf.setOperationSyncLogNum(
- Integer.parseInt(
- properties.getProperty(
- "operation_sync_log_file_num", String.valueOf(conf.getOperationSyncLogNum()))));
+ conf.setOperationSyncMaxLogSize(
+ Long.parseLong(
+ properties.getProperty(
+ "operation_sync_max_log_size", String.valueOf(conf.getOperationSyncMaxLogSize()))));
- conf.setOperationSyncMaxLogSize(
- Long.parseLong(
- properties.getProperty(
- "operation_sync_max_log_size",
- String.valueOf(conf.getOperationSyncMaxLogSize()))));
+ conf.setOperationSyncProducerCacheSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "operation_sync_producer_cache_size",
+ String.valueOf(conf.getOperationSyncProducerCacheSize()))));
- conf.setOperationSyncProducerCacheSize(
- Integer.parseInt(
- properties.getProperty(
- "operation_sync_producer_cache_size",
- String.valueOf(conf.getOperationSyncProducerCacheSize()))));
+ conf.setOperationSyncProducerCacheNum(
+ Integer.parseInt(
+ properties.getProperty(
+ "operation_sync_producer_cache_num",
+ String.valueOf(conf.getOperationSyncProducerCacheNum()))));
- conf.setOperationSyncProducerCacheNum(
- Integer.parseInt(
- properties.getProperty(
- "operation_sync_producer_cache_num",
- String.valueOf(conf.getOperationSyncProducerCacheNum()))));
+ conf.setSchemaQueryFetchSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "schema_query_fetch_size", String.valueOf(conf.getSchemaQueryFetchSize()))));
- conf.setSchemaQueryFetchSize(
- Integer.parseInt(
- properties.getProperty(
- "schema_query_fetch_size", String.valueOf(conf.getSchemaQueryFetchSize()))));
+ // At the same time, set TSFileConfig
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setTSFileStorageFs(
+ FSType.valueOf(
+ properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name())));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setDfsNameServices(properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setDfsHaNamenodes(
+ properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setDfsHaAutomaticFailoverEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "dfs_ha_automatic_failover_enabled",
+ String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setDfsClientFailoverProxyProvider(
+ properties.getProperty(
+ "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setUseKerberos(
+ Boolean.parseBoolean(
+ properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setKerberosKeytabFilePath(
+ properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath()));
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setKerberosPrincipal(
+ properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
+ TSFileDescriptor.getInstance().getConfig().setBatchSize(conf.getBatchSize());
- // At the same time, set TSFileConfig
- TSFileDescriptor.getInstance()
- .getConfig()
- .setTSFileStorageFs(
- FSType.valueOf(
- properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name())));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setHdfsPort(properties.getProperty("hdfs_port", conf.getHdfsPort()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setDfsNameServices(
- properties.getProperty("dfs_nameservices", conf.getDfsNameServices()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setDfsHaNamenodes(
- properties.getProperty("dfs_ha_namenodes", conf.getRawDfsHaNamenodes()).split(","));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setDfsHaAutomaticFailoverEnabled(
- Boolean.parseBoolean(
- properties.getProperty(
- "dfs_ha_automatic_failover_enabled",
- String.valueOf(conf.isDfsHaAutomaticFailoverEnabled()))));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setDfsClientFailoverProxyProvider(
- properties.getProperty(
- "dfs_client_failover_proxy_provider", conf.getDfsClientFailoverProxyProvider()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setUseKerberos(
- Boolean.parseBoolean(
- properties.getProperty(
- "hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setKerberosKeytabFilePath(
- properties.getProperty(
- "kerberos_keytab_file_path", conf.getKerberosKeytabFilePath()));
- TSFileDescriptor.getInstance()
- .getConfig()
- .setKerberosPrincipal(
- properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
- TSFileDescriptor.getInstance().getConfig().setBatchSize(conf.getBatchSize());
+ // timed flush memtable, timed close tsfile
+ loadTimedService(properties);
- // timed flush memtable, timed close tsfile
- loadTimedService(properties);
+ // set tsfile-format config
+ loadTsFileProps(properties);
- // set tsfile-format config
- loadTsFileProps(properties);
+ // make RPCTransportFactory taking effect.
+ RpcTransportFactory.reInit();
+
+ // UDF
+ loadUDFProps(properties);
- // make RPCTransportFactory taking effect.
- RpcTransportFactory.reInit();
+ // trigger
+ loadTriggerProps(properties);
- // UDF
- loadUDFProps(properties);
+ // CQ
+ loadCQProps(properties);
- // trigger
- loadTriggerProps(properties);
+ // external lib props
+ loadExternalLibProps(properties);
+ }
- // CQ
- loadCQProps(properties);
+ private void loadExternalLibProps(Properties properties) {
- } catch (FileNotFoundException e) {
- logger.warn("Fail to find config file {}", url, e);
- } catch (IOException e) {
- logger.warn("Cannot load config file, use default configuration", e);
- } catch (Exception e) {
- logger.warn("Incorrect format in config file, use default configuration", e);
- } finally {
- // update all data seriesPath
- conf.updatePath();
- // update instance in metric
- MetricConfigDescriptor.getInstance()
- .getMetricConfig()
- .updateRpcInstance(conf.getRpcAddress(), conf.getRpcPort());
- }
+ conf.setExternalPropertiesLoaderDir(
+ properties.getProperty(
+ "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir()));
+
+ conf.setExternalLimiterDir(
+ properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir()));
}
// to keep consistent with the cluster module.
diff --git a/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java
new file mode 100644
index 0000000000..482495eaa2
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/exception/metadata/SeriesNumberOverflowException.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.exception.metadata;
+
+import org.apache.iotdb.rpc.TSStatusCode;
+
+public class SeriesNumberOverflowException extends MetadataException {
+
+ public SeriesNumberOverflowException() {
+ super("exceed max allowed series number.", TSStatusCode.SERIES_OVERFLOW.getStatusCode());
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index b4fcd40d24..5689d599a3 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.exception.metadata.MetadataException;
import org.apache.iotdb.db.exception.metadata.NoTemplateOnMNodeException;
import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
import org.apache.iotdb.db.exception.metadata.PathNotExistException;
+import org.apache.iotdb.db.exception.metadata.SeriesNumberOverflowException;
import org.apache.iotdb.db.exception.metadata.StorageGroupAlreadySetException;
import org.apache.iotdb.db.exception.metadata.StorageGroupNotSetException;
import org.apache.iotdb.db.exception.metadata.TemplateIsInUseException;
@@ -104,6 +105,7 @@ import org.apache.iotdb.tsfile.write.schema.TimeseriesSchema;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.github.benmanes.caffeine.cache.LoadingCache;
+import org.apche.iotdb.external.api.ISeriesNumerLimiter;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.slf4j.Logger;
@@ -120,6 +122,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -186,8 +189,8 @@ public class MManager {
private boolean initialized;
private boolean allowToCreateNewSeries = true;
- private AtomicLong totalNormalSeriesNumber = new AtomicLong();
- private AtomicLong totalTemplateSeriesNumber = new AtomicLong();
+ private final AtomicLong totalNormalSeriesNumber = new AtomicLong();
+ private final AtomicLong totalTemplateSeriesNumber = new AtomicLong();
private final int mtreeSnapshotInterval;
private final long mtreeSnapshotThresholdTime;
@@ -205,6 +208,23 @@ public class MManager {
private TagManager tagManager = TagManager.getInstance();
private TemplateManager templateManager = TemplateManager.getInstance();
+ private ISeriesNumerLimiter seriesNumerLimiter =
+ new ISeriesNumerLimiter() {
+ @Override
+ public void init(Properties properties) {}
+
+ @Override
+ public boolean addTimeSeries(int number) {
+ // always return true, don't limit the number of series
+ return true;
+ }
+
+ @Override
+ public void deleteTimeSeries(int number) {
+ // do nothing
+ }
+ };
+
// region MManager Singleton
private static class MManagerHolder {
@@ -215,6 +235,10 @@ public class MManager {
private static final MManager INSTANCE = new MManager();
}
+ public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) {
+ this.seriesNumerLimiter = seriesNumerLimiter;
+ }
+
/** we should not use this function in other place, but only in IoTDB class */
public static MManager getInstance() {
return MManagerHolder.INSTANCE;
@@ -580,35 +604,47 @@ public class MManager {
"IoTDB system load is too large to create timeseries, "
+ "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
}
- try {
- PartialPath path = plan.getPath();
- SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding());
- ensureStorageGroup(path);
-
- TSDataType type = plan.getDataType();
- // create time series in MTree
- IMeasurementMNode leafMNode =
- mtree.createTimeseries(
- path,
- type,
- plan.getEncoding(),
- plan.getCompressor(),
- plan.getProps(),
- plan.getAlias());
-
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(path.getDevicePath());
-
- // update tag index
-
- if (offset != -1 && isRecovering) {
- // the timeseries has already been created and now system is recovering, using the tag info
- // in tagFile to recover index directly
- tagManager.recoverIndex(offset, leafMNode);
- } else if (plan.getTags() != null) {
- // tag key, tag value
- tagManager.addIndex(plan.getTags(), leafMNode);
+ if (!seriesNumerLimiter.addTimeSeries(1)) {
+ throw new SeriesNumberOverflowException();
+ }
+ try {
+ IMeasurementMNode leafMNode;
+ try {
+ PartialPath path = plan.getPath();
+ SchemaUtils.checkDataTypeWithEncoding(plan.getDataType(), plan.getEncoding());
+
+ ensureStorageGroup(path);
+
+ TSDataType type = plan.getDataType();
+ // create time series in MTree
+ leafMNode =
+ mtree.createTimeseries(
+ path,
+ type,
+ plan.getEncoding(),
+ plan.getCompressor(),
+ plan.getProps(),
+ plan.getAlias());
+
+ // the cached mNode may be replaced by new entityMNode in mtree
+ mNodeCache.invalidate(path.getDevicePath());
+
+ // update tag index
+
+ if (offset != -1 && isRecovering) {
+ // the timeseries has already been created and now system is recovering, using the tag
+ // info
+ // in tagFile to recover index directly
+ tagManager.recoverIndex(offset, leafMNode);
+ } else if (plan.getTags() != null) {
+ // tag key, tag value
+ tagManager.addIndex(plan.getTags(), leafMNode);
+ }
+ } catch (Throwable t) {
+ // roll back
+ seriesNumerLimiter.deleteTimeSeries(1);
+ throw t;
}
// update statistics and schemaDataTypeNumMap
@@ -692,31 +728,43 @@ public class MManager {
"IoTDB system load is too large to create timeseries, "
+ "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
}
+ int seriesCount = plan.getMeasurements().size();
+
+ if (!seriesNumerLimiter.addTimeSeries(seriesCount)) {
+ throw new SeriesNumberOverflowException();
+ }
+
try {
PartialPath prefixPath = plan.getPrefixPath();
List<String> measurements = plan.getMeasurements();
List<TSDataType> dataTypes = plan.getDataTypes();
List<TSEncoding> encodings = plan.getEncodings();
- for (int i = 0; i < measurements.size(); i++) {
- SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i));
- }
+ try {
+ for (int i = 0; i < measurements.size(); i++) {
+ SchemaUtils.checkDataTypeWithEncoding(dataTypes.get(i), encodings.get(i));
+ }
- ensureStorageGroup(prefixPath);
+ ensureStorageGroup(prefixPath);
- // create time series in MTree
- mtree.createAlignedTimeseries(
- prefixPath,
- measurements,
- plan.getDataTypes(),
- plan.getEncodings(),
- plan.getCompressors());
+ // create time series in MTree
+ mtree.createAlignedTimeseries(
+ prefixPath,
+ measurements,
+ plan.getDataTypes(),
+ plan.getEncodings(),
+ plan.getCompressors());
- // the cached mNode may be replaced by new entityMNode in mtree
- mNodeCache.invalidate(prefixPath);
+ // the cached mNode may be replaced by new entityMNode in mtree
+ mNodeCache.invalidate(prefixPath);
+ } catch (Throwable t) {
+ // roll back
+ seriesNumerLimiter.deleteTimeSeries(seriesCount);
+ throw t;
+ }
// update statistics and schemaDataTypeNumMap
- totalNormalSeriesNumber.addAndGet(measurements.size());
+ totalNormalSeriesNumber.addAndGet(seriesCount);
if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
logger.warn("Current series number {} is too large...", totalNormalSeriesNumber);
allowToCreateNewSeries = false;
@@ -844,6 +892,7 @@ public class MManager {
node = node.getParent();
}
totalNormalSeriesNumber.addAndGet(-1);
+ seriesNumerLimiter.deleteTimeSeries(1);
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber);
@@ -883,9 +932,11 @@ public class MManager {
public void deleteStorageGroups(List<PartialPath> storageGroups) throws MetadataException {
try {
for (PartialPath storageGroup : storageGroups) {
- totalNormalSeriesNumber.addAndGet(
- -mtree.getAllTimeseriesCount(
- storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false));
+ int timeSeriesCount =
+ mtree.getAllTimeseriesCount(
+ storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false);
+ totalNormalSeriesNumber.addAndGet(-timeSeriesCount);
+ seriesNumerLimiter.deleteTimeSeries(timeSeriesCount);
// clear cached MNode
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
@@ -2476,7 +2527,9 @@ public class MManager {
}
node.setUseTemplate(false);
- totalTemplateSeriesNumber.addAndGet(-node.getUpperTemplate().getMeasurementsCount());
+ int seriesCount = -node.getUpperTemplate().getMeasurementsCount();
+ totalTemplateSeriesNumber.addAndGet(-seriesCount);
+ seriesNumerLimiter.deleteTimeSeries(seriesCount);
// clear caches within MManger
mNodeCache.invalidate(node);
@@ -2502,24 +2555,36 @@ public class MManager {
String.format("Path [%s] has not been set any template.", node.getFullPath()));
}
- // this operation may change mtree structure and node type
- // invoke mnode.setUseTemplate is invalid
+ if (!seriesNumerLimiter.addTimeSeries(template.getMeasurementsCount())) {
+ throw new SeriesNumberOverflowException();
+ }
+
+ IMNode mountedMNode;
+ try {
+ // this operation may change mtree structure and node type
+ // invoke mnode.setUseTemplate is invalid
- // check alignment of template and mounted node
- // if direct measurement exists, node will be replaced
- IMNode mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template);
+ // check alignment of template and mounted node
+ // if direct measurement exists, node will be replaced
+ mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template);
- // if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or
- // template
- if (mountedMNode.isEntity()) {
- mountedMNode
- .getAsEntityMNode()
- .setAligned(
- node.isEntity()
- ? node.getAsEntityMNode().isAligned()
- : node.getUpperTemplate().isDirectAligned());
+ // if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or
+ // template
+ if (mountedMNode.isEntity()) {
+ mountedMNode
+ .getAsEntityMNode()
+ .setAligned(
+ node.isEntity()
+ ? node.getAsEntityMNode().isAligned()
+ : node.getUpperTemplate().isDirectAligned());
+ }
+ mountedMNode.setUseTemplate(true);
+ } catch (Throwable t) {
+ // roll back
+ seriesNumerLimiter.deleteTimeSeries(template.getMeasurementsCount());
+ throw t;
}
- mountedMNode.setUseTemplate(true);
+
totalTemplateSeriesNumber.addAndGet(template.getMeasurementsCount());
if (node != mountedMNode) {
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index fcd152f612..66ffc23ac4 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -53,6 +53,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
+import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib;
+
public class IoTDB implements IoTDBMBean {
private static final Logger logger = LoggerFactory.getLogger(IoTDB.class);
@@ -78,6 +80,9 @@ public class IoTDB implements IoTDBMBean {
System.exit(1);
}
IoTDB daemon = IoTDB.getInstance();
+
+ loadExternLib(config);
+
daemon.active();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
new file mode 100644
index 0000000000..a2bfd94cc9
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.utils;
+
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+
+import org.apache.commons.io.FileUtils;
+import org.apche.iotdb.external.api.IPropertiesLoader;
+import org.apche.iotdb.external.api.ISeriesNumerLimiter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.net.UnknownHostException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.ServiceLoader;
+
+public class JarLoaderUtil {
+
+ private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class);
+
+ public static URL[] getExternalJarURLs(String jarDir) throws IOException {
+ HashSet<File> fileSet =
+ new HashSet<>(
+ FileUtils.listFiles(
+ SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true));
+ return FileUtils.toURLs(fileSet.toArray(new File[0]));
+ }
+
+ public static void loadExternLib(IoTDBConfig config) {
+ // load external properties
+ String loaderDir = config.getExternalPropertiesLoaderDir();
+
+ if (!(new File(loaderDir).exists())) {
+ return;
+ }
+
+ Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath();
+ URL[] loaderJarURLs;
+ List<Properties> externalPropertiesList = new ArrayList<>();
+ try {
+ loaderJarURLs = getExternalJarURLs(loaderDir);
+
+ if (loaderJarURLs == null || loaderJarURLs.length == 0) {
+ return;
+ }
+
+ ClassLoader classLoader = new URLClassLoader(loaderJarURLs);
+
+ // Use SPI to get all plugins' class
+ ServiceLoader<IPropertiesLoader> loaders =
+ ServiceLoader.load(IPropertiesLoader.class, classLoader);
+
+ for (IPropertiesLoader loader : loaders) {
+ if (loader == null) {
+ logger.error("IPropertiesLoader(), loader is null.");
+ continue;
+ }
+ Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath());
+ if (properties != null) {
+ externalPropertiesList.add(properties);
+ }
+ }
+ } catch (Throwable t) {
+ logger.error("error happened while loading external loader. ", t);
+ // ignore
+ }
+
+ if (externalPropertiesList.size() != 1) {
+ return;
+ }
+
+ // overwrite the default properties;
+ for (Properties properties : externalPropertiesList) {
+ try {
+ IoTDBDescriptor.getInstance().loadProperties(properties);
+ } catch (UnknownHostException e) {
+ // ignore
+ logger.error("error happened while loading external properties. ", e);
+ }
+ TSFileDescriptor.getInstance()
+ .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties);
+ }
+
+ String limiterDir = config.getExternalLimiterDir();
+
+ if (!(new File(loaderDir).exists())) {
+ return;
+ }
+
+ URL[] limiterJarURLs;
+
+ List<ISeriesNumerLimiter> limiterList = new ArrayList<>();
+
+ try {
+ limiterJarURLs = getExternalJarURLs(limiterDir);
+
+ if (limiterJarURLs == null || limiterJarURLs.length == 0) {
+ return;
+ }
+
+ ClassLoader classLoader = new URLClassLoader(limiterJarURLs);
+
+ // Use SPI to get all plugins' class
+ ServiceLoader<ISeriesNumerLimiter> limiters =
+ ServiceLoader.load(ISeriesNumerLimiter.class, classLoader);
+
+ for (ISeriesNumerLimiter limiter : limiters) {
+ if (limiter == null) {
+ logger.error("ISeriesNumerLimiter(), limiter is null.");
+ continue;
+ }
+ for (Properties properties : externalPropertiesList) {
+ limiter.init(properties);
+ }
+ limiterList.add(limiter);
+ }
+ } catch (Throwable t) {
+ // ignore
+ logger.error("error happened while loading external limiter. ", t);
+ }
+
+ if (limiterList.size() != 1) {
+ return;
+ }
+
+ MManager.getInstance().setSeriesNumerLimiter(limiterList.get(0));
+ }
+}
diff --git a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
index 0de8ca7779..e330d04731 100644
--- a/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
+++ b/service-rpc/src/main/java/org/apache/iotdb/rpc/TSStatusCode.java
@@ -59,6 +59,8 @@ public enum TSStatusCode {
TEMPLATE_IS_IN_USE(326),
TEMPLATE_IMCOMPATIBLE(327),
+ SERIES_OVERFLOW(337),
+
EXECUTE_STATEMENT_ERROR(400),
SQL_PARSE_ERROR(401),
GENERATE_TIME_ZONE_ERROR(402),
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 840e64d756..711c5a4fd2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -61,7 +61,7 @@ public class TSFileDescriptor {
}
}
- private void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) {
+ public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) {
PropertiesOverWriter writer = new PropertiesOverWriter(properties);
writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");