You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2022/08/30 17:07:43 UTC
[iotdb] branch re-ext-lib-0.13 created (now ae15681754)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch re-ext-lib-0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at ae15681754 refactor the mechanism of using external-api
This branch includes the following new commits:
new ae15681754 refactor the mechanism of using external-api
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: refactor the mechanism of using external-api
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch re-ext-lib-0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae156817544481cab463ecc9cf6c4570c2e9e193
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Wed Aug 31 01:07:17 2022 +0800
refactor the mechanism of using external-api
---
.../iotdb/external/api/IPropertiesLoader.java | 11 +-
...sNumerLimiter.java => ISeriesNumerMonitor.java} | 7 +-
.../resources/conf/iotdb-engine.properties | 25 +---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 41 ++----
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 79 +++--------
.../org/apache/iotdb/db/metadata/MManager.java | 69 +++++----
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 -
.../org/apache/iotdb/db/utils/JarLoaderUtil.java | 156 ---------------------
.../iotdb/tsfile/common/conf/TSFileConfig.java | 11 ++
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +-
11 files changed, 94 insertions(+), 317 deletions(-)
diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
index 4e62929c15..5cf90fc950 100644
--- a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
+++ b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.external.api;
-import java.nio.file.Path;
import java.util.Properties;
/**
@@ -30,8 +29,14 @@ public interface IPropertiesLoader {
/**
* Load Properties from specific file
*
- * @param file The path of the properties file to open
* @return a property list with values in file.
*/
- Properties loadProperties(Path file);
+ Properties loadProperties();
+
+ /**
+ * some properties that iotdb does not use by default.
+ *
+ * @return properties.
+ */
+ Properties getCustomizedProperties();
}
diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
similarity index 82%
rename from external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java
rename to external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
index 2a740ff1ae..acbc2bbfa2 100644
--- a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java
+++ b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
@@ -20,8 +20,8 @@ package org.apache.iotdb.external.api;
import java.util.Properties;
-/** An interface for series number limiting, users can implement their own limitation strategy */
-public interface ISeriesNumerLimiter {
+/** An interface for series number monitoring, users can implement their own limitation strategy */
+public interface ISeriesNumerMonitor {
/**
* do the necessary initialization
@@ -34,8 +34,7 @@ public interface ISeriesNumerLimiter {
* add time series
*
* @param number time series number for current createTimeSeries operation
- * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries
- * operation is allowed, otherwise false
+ * @return true if we want to allow the operation, otherwise false
*/
boolean addTimeSeries(int number);
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index ebd1a35242..0be00a433d 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -935,27 +935,4 @@ timestamp_precision=ms
### Group By Fill Configuration
####################
# Datatype: float
-# group_by_fill_cache_size_in_mb=1.0
-
-
-####################
-### External Lib Configuration
-####################
-
-# external lib directory for properties loader
-# For Window platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
-# absolute. Otherwise, it is relative.
-# external_properties_loader_dir=ext\\loader
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# external_properties_loader_dir=ext/loader
-
-# external lib directory for limiter
-# For Window platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
-# absolute. Otherwise, it is relative.
-# external_limiter_dir=ext\\limiter
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# external_limiter_dir=ext/limiter
\ No newline at end of file
+# group_by_fill_cache_size_in_mb=1.0
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c3e0bdcde9..74cb6fa4aa 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,7 +50,6 @@ public class IoTDBConfig {
/* Names of Watermark methods */
public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod";
static final String CONFIG_NAME = "iotdb-engine.properties";
- public static final String EXTERNAL_CONFIG_NAME = "iotdb-engine-external.properties";
private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class);
private static final String MULTI_DIR_STRATEGY_PREFIX =
"org.apache.iotdb.db.conf.directories.strategy.";
@@ -246,16 +246,6 @@ public class IoTDBConfig {
private String triggerDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TRIGGER_FOLDER_NAME;
- /** External lib directory for properties loader, stores user-uploaded JAR files */
- private String externalPropertiesLoaderDir =
- IoTDBConstant.EXT_FOLDER_NAME
- + File.separator
- + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME;
-
- /** External lib directory for limiter, stores user uploaded JAR files */
- private String externalLimiterDir =
- IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER;
-
/** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */
private String[] dataDirs = {"data" + File.separator + "data"};
@@ -865,6 +855,9 @@ public class IoTDBConfig {
// The max record num returned in one schema query.
private int schemaQueryFetchSize = 10000000;
+ // customizedProperties, this should be empty by default.
+ private Properties customizedProperties = new Properties();
+
public IoTDBConfig() {
// empty constructor
}
@@ -999,8 +992,6 @@ public class IoTDBConfig {
udfDir = addHomeDir(udfDir);
triggerDir = addHomeDir(triggerDir);
operationSyncLogDir = addHomeDir(operationSyncLogDir);
- externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir);
- externalLimiterDir = addHomeDir(externalLimiterDir);
if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) {
String hdfsDir = getHdfsDir();
@@ -1225,22 +1216,6 @@ public class IoTDBConfig {
this.triggerDir = triggerDir;
}
- public String getExternalPropertiesLoaderDir() {
- return externalPropertiesLoaderDir;
- }
-
- public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) {
- this.externalPropertiesLoaderDir = externalPropertiesLoaderDir;
- }
-
- public String getExternalLimiterDir() {
- return externalLimiterDir;
- }
-
- public void setExternalLimiterDir(String externalLimiterDir) {
- this.externalLimiterDir = externalLimiterDir;
- }
-
public String getMultiDirStrategyClassName() {
return multiDirStrategyClassName;
}
@@ -2765,4 +2740,12 @@ public class IoTDBConfig {
public void setSchemaQueryFetchSize(int schemaQueryFetchSize) {
this.schemaQueryFetchSize = schemaQueryFetchSize;
}
+
+ public Properties getCustomizedProperties() {
+ return customizedProperties;
+ }
+
+ public void setCustomizedProperties(Properties customizedProperties) {
+ this.customizedProperties = customizedProperties;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 337a9e0a6e..1294002707 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -191,10 +191,6 @@ public class IoTDBConstant {
// compaction mods of previous version (<0.13)
public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods";
- public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader";
-
- public static final String EXT_LIMITER = "limiter";
-
// client version number
public enum ClientVersion {
V_0_12,
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index cdb0057a2d..05e400ac87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.service.metrics.MetricService;
+import org.apache.iotdb.external.api.IPropertiesLoader;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.config.ReloadLevel;
import org.apache.iotdb.rpc.RpcTransportFactory;
@@ -45,12 +46,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Properties;
+import java.util.ServiceLoader;
public class IoTDBDescriptor {
@@ -60,6 +59,21 @@ public class IoTDBDescriptor {
protected IoTDBDescriptor() {
loadProps();
+ ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader =
+ ServiceLoader.load(IPropertiesLoader.class);
+ for (IPropertiesLoader loader : propertiesLoaderServiceLoader) {
+ try {
+ Properties properties = loader.loadProperties();
+ loadProperties(properties);
+ conf.setCustomizedProperties(loader.getCustomizedProperties());
+ TSFileDescriptor.getInstance().overwriteConfigByCustomSettings(properties);
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setCustomizedProperties(loader.getCustomizedProperties());
+ } catch (UnknownHostException e) {
+ logger.warn("load property from {} failed.", loader.getClass().getName(), e);
+ }
+ }
}
public static IoTDBDescriptor getInstance() {
@@ -117,52 +131,6 @@ public class IoTDBDescriptor {
}
}
- /**
- * get external props url location
- *
- * @return url object if location exit, otherwise null.
- */
- public Path getExternalPropsPath() {
- // Check if a config-directory was specified first.
- String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
- // If it wasn't, check if a home directory was provided (This usually contains a config)
- if (urlString == null) {
- urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
- if (urlString != null) {
- urlString =
- urlString
- + File.separatorChar
- + "conf"
- + File.separatorChar
- + IoTDBConfig.EXTERNAL_CONFIG_NAME;
- } else {
- // If this too wasn't provided, try to find a default config in the root of the classpath.
- URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME);
- if (uri != null) {
- try {
- return Paths.get(uri.toURI());
- } catch (URISyntaxException e) {
- return null;
- }
- }
- logger.warn(
- "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading "
- + "config file {}, use default configuration",
- IoTDBConfig.EXTERNAL_CONFIG_NAME);
- // update all data seriesPath
- conf.updatePath();
- return null;
- }
- }
- // If a config location was provided, but it doesn't end with a properties file,
- // append the default location.
- else if (!urlString.endsWith(".properties")) {
- urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME);
- }
-
- return Paths.get(urlString);
- }
-
/** load an property file and set TsfileDBConfig variables. */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private void loadProps() {
@@ -976,19 +944,6 @@ public class IoTDBDescriptor {
// CQ
loadCQProps(properties);
-
- // external lib props
- loadExternalLibProps(properties);
- }
-
- private void loadExternalLibProps(Properties properties) {
-
- conf.setExternalPropertiesLoaderDir(
- properties.getProperty(
- "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir()));
-
- conf.setExternalLimiterDir(
- properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir()));
}
// to keep consistent with the cluster module.
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0f16a467c4..8f429f50ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -91,7 +91,7 @@ import org.apache.iotdb.db.service.metrics.enums.Tag;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
+import org.apache.iotdb.external.api.ISeriesNumerMonitor;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -122,7 +122,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
+import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -208,22 +208,8 @@ public class MManager {
private TagManager tagManager = TagManager.getInstance();
private TemplateManager templateManager = TemplateManager.getInstance();
- private ISeriesNumerLimiter seriesNumerLimiter =
- new ISeriesNumerLimiter() {
- @Override
- public void init(Properties properties) {}
-
- @Override
- public boolean addTimeSeries(int number) {
- // always return true, don't limit the number of series
- return true;
- }
-
- @Override
- public void deleteTimeSeries(int number) {
- // do nothing
- }
- };
+ // seriesNumerLimiter may be null, so we must check it before use it.
+ private ISeriesNumerMonitor seriesNumerMonitor = null;
// region MManager Singleton
private static class MManagerHolder {
@@ -235,8 +221,8 @@ public class MManager {
private static final MManager INSTANCE = new MManager();
}
- public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) {
- this.seriesNumerLimiter = seriesNumerLimiter;
+ public void setSeriesNumerMonitor(ISeriesNumerMonitor seriesNumerMonitor) {
+ this.seriesNumerMonitor = seriesNumerMonitor;
}
/** we should not use this function in other place, but only in IoTDB class */
@@ -247,6 +233,18 @@ public class MManager {
// region Interfaces and Implementation of MManager initialization、snapshot、recover and clear
protected MManager() {
+ // init seriesNumerLimiter if there is.
+ // each mmanager instance will generate an ISeriesNumerLimiter instance
+ // So, if you want to share the ISeriesNumerLimiter instance, pls change this part of code.
+ ServiceLoader<ISeriesNumerMonitor> limiterServiceLoader =
+ ServiceLoader.load(ISeriesNumerMonitor.class);
+ for (ISeriesNumerMonitor loader : limiterServiceLoader) {
+ if (this.seriesNumerMonitor != null) {
+ // it means there is more than one ISeriesNumerLimiter implementation.
+ logger.warn("There are more than one ISeriesNumerLimiter implementation. pls check.");
+ }
+ this.seriesNumerMonitor = loader;
+ }
mtreeSnapshotInterval = config.getMtreeSnapshotInterval();
mtreeSnapshotThresholdTime = config.getMtreeSnapshotThresholdTime() * 1000L;
String schemaDir = config.getSchemaDir();
@@ -605,7 +603,7 @@ public class MManager {
+ "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
}
- if (!seriesNumerLimiter.addTimeSeries(1)) {
+ if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) {
throw new SeriesNumberOverflowException();
}
try {
@@ -643,7 +641,9 @@ public class MManager {
}
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(1);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(1);
+ }
throw t;
}
@@ -730,7 +730,7 @@ public class MManager {
}
int seriesCount = plan.getMeasurements().size();
- if (!seriesNumerLimiter.addTimeSeries(seriesCount)) {
+ if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) {
throw new SeriesNumberOverflowException();
}
@@ -759,7 +759,9 @@ public class MManager {
mNodeCache.invalidate(prefixPath);
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(seriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(seriesCount);
+ }
throw t;
}
@@ -892,7 +894,9 @@ public class MManager {
node = node.getParent();
}
totalNormalSeriesNumber.addAndGet(-1);
- seriesNumerLimiter.deleteTimeSeries(1);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(1);
+ }
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber);
@@ -936,7 +940,9 @@ public class MManager {
mtree.getAllTimeseriesCount(
storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false);
totalNormalSeriesNumber.addAndGet(-timeSeriesCount);
- seriesNumerLimiter.deleteTimeSeries(timeSeriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(timeSeriesCount);
+ }
// clear cached MNode
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
@@ -2529,7 +2535,9 @@ public class MManager {
node.setUseTemplate(false);
int seriesCount = node.getUpperTemplate().getMeasurementsCount();
totalTemplateSeriesNumber.addAndGet(-seriesCount);
- seriesNumerLimiter.deleteTimeSeries(seriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(seriesCount);
+ }
// clear caches within MManger
mNodeCache.invalidate(node);
@@ -2555,7 +2563,8 @@ public class MManager {
String.format("Path [%s] has not been set any template.", node.getFullPath()));
}
- if (!seriesNumerLimiter.addTimeSeries(template.getMeasurementsCount())) {
+ if (seriesNumerMonitor != null
+ && !seriesNumerMonitor.addTimeSeries(template.getMeasurementsCount())) {
throw new SeriesNumberOverflowException();
}
@@ -2581,7 +2590,9 @@ public class MManager {
mountedMNode.setUseTemplate(true);
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(template.getMeasurementsCount());
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(template.getMeasurementsCount());
+ }
throw t;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 66ffc23ac4..0c9c9c7dd2 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -53,8 +53,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
-import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib;
-
public class IoTDB implements IoTDBMBean {
private static final Logger logger = LoggerFactory.getLogger(IoTDB.class);
@@ -81,8 +79,6 @@ public class IoTDB implements IoTDBMBean {
}
IoTDB daemon = IoTDB.getInstance();
- loadExternLib(config);
-
daemon.active();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
deleted file mode 100644
index 30d0c28f47..0000000000
--- a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.utils;
-
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.external.api.IPropertiesLoader;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.net.UnknownHostException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.ServiceLoader;
-
-public class JarLoaderUtil {
-
- private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class);
-
- public static URL[] getExternalJarURLs(String jarDir) throws IOException {
- HashSet<File> fileSet =
- new HashSet<>(
- FileUtils.listFiles(
- SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true));
- return FileUtils.toURLs(fileSet.toArray(new File[0]));
- }
-
- public static void loadExternLib(IoTDBConfig config) {
- // load external properties
- String loaderDir = config.getExternalPropertiesLoaderDir();
-
- if (!(new File(loaderDir).exists())) {
- return;
- }
-
- Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath();
- URL[] loaderJarURLs;
- List<Properties> externalPropertiesList = new ArrayList<>();
- try {
- loaderJarURLs = getExternalJarURLs(loaderDir);
-
- if (loaderJarURLs == null || loaderJarURLs.length == 0) {
- return;
- }
-
- ClassLoader classLoader = new URLClassLoader(loaderJarURLs);
-
- // Use SPI to get all plugins' class
- ServiceLoader<IPropertiesLoader> loaders =
- ServiceLoader.load(IPropertiesLoader.class, classLoader);
-
- for (IPropertiesLoader loader : loaders) {
- if (loader == null) {
- logger.error("IPropertiesLoader(), loader is null.");
- continue;
- }
- Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath());
- if (properties != null) {
- externalPropertiesList.add(properties);
- }
- }
- } catch (Throwable t) {
- logger.error("error happened while loading external loader. ", t);
- // ignore
- }
-
- if (externalPropertiesList.size() != 1) {
- return;
- }
-
- // overwrite the default properties;
- for (Properties properties : externalPropertiesList) {
- try {
- IoTDBDescriptor.getInstance().loadProperties(properties);
- } catch (UnknownHostException e) {
- // ignore
- logger.error("error happened while loading external properties. ", e);
- }
- TSFileDescriptor.getInstance()
- .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties);
- }
-
- String limiterDir = config.getExternalLimiterDir();
-
- if (!(new File(loaderDir).exists())) {
- return;
- }
-
- URL[] limiterJarURLs;
-
- List<ISeriesNumerLimiter> limiterList = new ArrayList<>();
-
- try {
- limiterJarURLs = getExternalJarURLs(limiterDir);
-
- if (limiterJarURLs == null || limiterJarURLs.length == 0) {
- return;
- }
-
- ClassLoader classLoader = new URLClassLoader(limiterJarURLs);
-
- // Use SPI to get all plugins' class
- ServiceLoader<ISeriesNumerLimiter> limiters =
- ServiceLoader.load(ISeriesNumerLimiter.class, classLoader);
-
- for (ISeriesNumerLimiter limiter : limiters) {
- if (limiter == null) {
- logger.error("ISeriesNumerLimiter(), limiter is null.");
- continue;
- }
- for (Properties properties : externalPropertiesList) {
- limiter.init(properties);
- }
- limiterList.add(limiter);
- }
- } catch (Throwable t) {
- // ignore
- logger.error("error happened while loading external limiter. ", t);
- }
-
- if (limiterList.size() != 1) {
- return;
- }
-
- MManager.getInstance().setSeriesNumerLimiter(limiterList.get(0));
- }
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 060aba9189..df5df28446 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.fileSystem.FSType;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Properties;
/** TSFileConfig is a configuration class. Every variable is public and has default value. */
public class TSFileConfig implements Serializable {
@@ -145,6 +146,8 @@ public class TSFileConfig implements Serializable {
private double bloomFilterErrorRate = 0.05;
/** The amount of data iterate each time */
private int batchSize = 1000;
+ /** customizedProperties, this should be empty by default. */
+ private Properties customizedProperties = new Properties();
public TSFileConfig() {}
@@ -411,4 +414,12 @@ public class TSFileConfig implements Serializable {
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
+
+ public Properties getCustomizedProperties() {
+ return customizedProperties;
+ }
+
+ public void setCustomizedProperties(Properties customizedProperties) {
+ this.customizedProperties = customizedProperties;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 711c5a4fd2..765bea8684 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -57,11 +57,11 @@ public class TSFileDescriptor {
private void init() {
Properties properties = loadProperties();
if (properties != null) {
- overwriteConfigByCustomSettings(this.conf, properties);
+ overwriteConfigByCustomSettings(properties);
}
}
- public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) {
+ public void overwriteConfigByCustomSettings(Properties properties) {
PropertiesOverWriter writer = new PropertiesOverWriter(properties);
writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");