You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2022/08/30 17:07:44 UTC
[iotdb] 01/01: refactor the mechanism of using external-api
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch re-ext-lib-0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit ae156817544481cab463ecc9cf6c4570c2e9e193
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Wed Aug 31 01:07:17 2022 +0800
refactor the mechanism of using external-api
---
.../iotdb/external/api/IPropertiesLoader.java | 11 +-
...sNumerLimiter.java => ISeriesNumerMonitor.java} | 7 +-
.../resources/conf/iotdb-engine.properties | 25 +---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 41 ++----
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 4 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 79 +++--------
.../org/apache/iotdb/db/metadata/MManager.java | 69 +++++----
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 -
.../org/apache/iotdb/db/utils/JarLoaderUtil.java | 156 ---------------------
.../iotdb/tsfile/common/conf/TSFileConfig.java | 11 ++
.../iotdb/tsfile/common/conf/TSFileDescriptor.java | 4 +-
11 files changed, 94 insertions(+), 317 deletions(-)
diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
index 4e62929c15..5cf90fc950 100644
--- a/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
+++ b/external-api/src/main/java/org/apache/iotdb/external/api/IPropertiesLoader.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.external.api;
-import java.nio.file.Path;
import java.util.Properties;
/**
@@ -30,8 +29,14 @@ public interface IPropertiesLoader {
/**
* Load Properties from specific file
*
- * @param file The path of the properties file to open
* @return a property list with values in file.
*/
- Properties loadProperties(Path file);
+ Properties loadProperties();
+
+ /**
+ * some properties that iotdb does not use by default.
+ *
+ * @return properties.
+ */
+ Properties getCustomizedProperties();
}
diff --git a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
similarity index 82%
rename from external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java
rename to external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
index 2a740ff1ae..acbc2bbfa2 100644
--- a/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerLimiter.java
+++ b/external-api/src/main/java/org/apache/iotdb/external/api/ISeriesNumerMonitor.java
@@ -20,8 +20,8 @@ package org.apache.iotdb.external.api;
import java.util.Properties;
-/** An interface for series number limiting, users can implement their own limitation strategy */
-public interface ISeriesNumerLimiter {
+/** An interface for series number monitoring, users can implement their own limitation strategy */
+public interface ISeriesNumerMonitor {
/**
* do the necessary initialization
@@ -34,8 +34,7 @@ public interface ISeriesNumerLimiter {
* add time series
*
* @param number time series number for current createTimeSeries operation
- * @return true if totalTimeSeriesNumber doesn't exceed the limit and current createTimeSeries
- * operation is allowed, otherwise false
+ * @return true if we want to allow the operation, otherwise false
*/
boolean addTimeSeries(int number);
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index ebd1a35242..0be00a433d 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -935,27 +935,4 @@ timestamp_precision=ms
### Group By Fill Configuration
####################
# Datatype: float
-# group_by_fill_cache_size_in_mb=1.0
-
-
-####################
-### External Lib Configuration
-####################
-
-# external lib directory for properties loader
-# For Window platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
-# absolute. Otherwise, it is relative.
-# external_properties_loader_dir=ext\\loader
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# external_properties_loader_dir=ext/loader
-
-# external lib directory for limiter
-# For Window platform
-# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is
-# absolute. Otherwise, it is relative.
-# external_limiter_dir=ext\\limiter
-# For Linux platform
-# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# external_limiter_dir=ext/limiter
\ No newline at end of file
+# group_by_fill_cache_size_in_mb=1.0
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index c3e0bdcde9..74cb6fa4aa 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
+import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -49,7 +50,6 @@ public class IoTDBConfig {
/* Names of Watermark methods */
public static final String WATERMARK_GROUPED_LSB = "GroupBasedLSBMethod";
static final String CONFIG_NAME = "iotdb-engine.properties";
- public static final String EXTERNAL_CONFIG_NAME = "iotdb-engine-external.properties";
private static final Logger logger = LoggerFactory.getLogger(IoTDBConfig.class);
private static final String MULTI_DIR_STRATEGY_PREFIX =
"org.apache.iotdb.db.conf.directories.strategy.";
@@ -246,16 +246,6 @@ public class IoTDBConfig {
private String triggerDir =
IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.TRIGGER_FOLDER_NAME;
- /** External lib directory for properties loader, stores user-uploaded JAR files */
- private String externalPropertiesLoaderDir =
- IoTDBConstant.EXT_FOLDER_NAME
- + File.separator
- + IoTDBConstant.EXT_PROPERTIES_LOADER_FOLDER_NAME;
-
- /** External lib directory for limiter, stores user uploaded JAR files */
- private String externalLimiterDir =
- IoTDBConstant.EXT_FOLDER_NAME + File.separator + IoTDBConstant.EXT_LIMITER;
-
/** Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"}; */
private String[] dataDirs = {"data" + File.separator + "data"};
@@ -865,6 +855,9 @@ public class IoTDBConfig {
// The max record num returned in one schema query.
private int schemaQueryFetchSize = 10000000;
+ // customizedProperties, this should be empty by default.
+ private Properties customizedProperties = new Properties();
+
public IoTDBConfig() {
// empty constructor
}
@@ -999,8 +992,6 @@ public class IoTDBConfig {
udfDir = addHomeDir(udfDir);
triggerDir = addHomeDir(triggerDir);
operationSyncLogDir = addHomeDir(operationSyncLogDir);
- externalPropertiesLoaderDir = addHomeDir(externalPropertiesLoaderDir);
- externalLimiterDir = addHomeDir(externalLimiterDir);
if (TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs().equals(FSType.HDFS)) {
String hdfsDir = getHdfsDir();
@@ -1225,22 +1216,6 @@ public class IoTDBConfig {
this.triggerDir = triggerDir;
}
- public String getExternalPropertiesLoaderDir() {
- return externalPropertiesLoaderDir;
- }
-
- public void setExternalPropertiesLoaderDir(String externalPropertiesLoaderDir) {
- this.externalPropertiesLoaderDir = externalPropertiesLoaderDir;
- }
-
- public String getExternalLimiterDir() {
- return externalLimiterDir;
- }
-
- public void setExternalLimiterDir(String externalLimiterDir) {
- this.externalLimiterDir = externalLimiterDir;
- }
-
public String getMultiDirStrategyClassName() {
return multiDirStrategyClassName;
}
@@ -2765,4 +2740,12 @@ public class IoTDBConfig {
public void setSchemaQueryFetchSize(int schemaQueryFetchSize) {
this.schemaQueryFetchSize = schemaQueryFetchSize;
}
+
+ public Properties getCustomizedProperties() {
+ return customizedProperties;
+ }
+
+ public void setCustomizedProperties(Properties customizedProperties) {
+ this.customizedProperties = customizedProperties;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 337a9e0a6e..1294002707 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -191,10 +191,6 @@ public class IoTDBConstant {
// compaction mods of previous version (<0.13)
public static final String COMPACTION_MODIFICATION_FILE_NAME_FROM_OLD = "merge.mods";
- public static final String EXT_PROPERTIES_LOADER_FOLDER_NAME = "loader";
-
- public static final String EXT_LIMITER = "limiter";
-
// client version number
public enum ClientVersion {
V_0_12,
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index cdb0057a2d..05e400ac87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.engine.compaction.inner.InnerCompactionStrategy;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.service.metrics.MetricService;
+import org.apache.iotdb.external.api.IPropertiesLoader;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.config.ReloadLevel;
import org.apache.iotdb.rpc.RpcTransportFactory;
@@ -45,12 +46,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.MalformedURLException;
-import java.net.URISyntaxException;
import java.net.URL;
import java.net.UnknownHostException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.Properties;
+import java.util.ServiceLoader;
public class IoTDBDescriptor {
@@ -60,6 +59,21 @@ public class IoTDBDescriptor {
protected IoTDBDescriptor() {
loadProps();
+ ServiceLoader<IPropertiesLoader> propertiesLoaderServiceLoader =
+ ServiceLoader.load(IPropertiesLoader.class);
+ for (IPropertiesLoader loader : propertiesLoaderServiceLoader) {
+ try {
+ Properties properties = loader.loadProperties();
+ loadProperties(properties);
+ conf.setCustomizedProperties(loader.getCustomizedProperties());
+ TSFileDescriptor.getInstance().overwriteConfigByCustomSettings(properties);
+ TSFileDescriptor.getInstance()
+ .getConfig()
+ .setCustomizedProperties(loader.getCustomizedProperties());
+ } catch (UnknownHostException e) {
+ logger.warn("load property from {} failed.", loader.getClass().getName(), e);
+ }
+ }
}
public static IoTDBDescriptor getInstance() {
@@ -117,52 +131,6 @@ public class IoTDBDescriptor {
}
}
- /**
- * get external props url location
- *
- * @return url object if location exit, otherwise null.
- */
- public Path getExternalPropsPath() {
- // Check if a config-directory was specified first.
- String urlString = System.getProperty(IoTDBConstant.IOTDB_CONF, null);
- // If it wasn't, check if a home directory was provided (This usually contains a config)
- if (urlString == null) {
- urlString = System.getProperty(IoTDBConstant.IOTDB_HOME, null);
- if (urlString != null) {
- urlString =
- urlString
- + File.separatorChar
- + "conf"
- + File.separatorChar
- + IoTDBConfig.EXTERNAL_CONFIG_NAME;
- } else {
- // If this too wasn't provided, try to find a default config in the root of the classpath.
- URL uri = IoTDBConfig.class.getResource("/" + IoTDBConfig.EXTERNAL_CONFIG_NAME);
- if (uri != null) {
- try {
- return Paths.get(uri.toURI());
- } catch (URISyntaxException e) {
- return null;
- }
- }
- logger.warn(
- "Cannot find IOTDB_HOME or IOTDB_EXTERNAL_CONF environment variable when loading "
- + "config file {}, use default configuration",
- IoTDBConfig.EXTERNAL_CONFIG_NAME);
- // update all data seriesPath
- conf.updatePath();
- return null;
- }
- }
- // If a config location was provided, but it doesn't end with a properties file,
- // append the default location.
- else if (!urlString.endsWith(".properties")) {
- urlString += (File.separatorChar + IoTDBConfig.EXTERNAL_CONFIG_NAME);
- }
-
- return Paths.get(urlString);
- }
-
/** load an property file and set TsfileDBConfig variables. */
@SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity warning
private void loadProps() {
@@ -976,19 +944,6 @@ public class IoTDBDescriptor {
// CQ
loadCQProps(properties);
-
- // external lib props
- loadExternalLibProps(properties);
- }
-
- private void loadExternalLibProps(Properties properties) {
-
- conf.setExternalPropertiesLoaderDir(
- properties.getProperty(
- "external_properties_loader_dir", conf.getExternalPropertiesLoaderDir()));
-
- conf.setExternalLimiterDir(
- properties.getProperty("external_limiter_dir", conf.getExternalLimiterDir()));
}
// to keep consistent with the cluster module.
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 0f16a467c4..8f429f50ad 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -91,7 +91,7 @@ import org.apache.iotdb.db.service.metrics.enums.Tag;
import org.apache.iotdb.db.utils.SchemaUtils;
import org.apache.iotdb.db.utils.TestOnly;
import org.apache.iotdb.db.utils.TypeInferenceUtils;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
+import org.apache.iotdb.external.api.ISeriesNumerMonitor;
import org.apache.iotdb.metrics.config.MetricConfigDescriptor;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -122,7 +122,7 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Properties;
+import java.util.ServiceLoader;
import java.util.Set;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -208,22 +208,8 @@ public class MManager {
private TagManager tagManager = TagManager.getInstance();
private TemplateManager templateManager = TemplateManager.getInstance();
- private ISeriesNumerLimiter seriesNumerLimiter =
- new ISeriesNumerLimiter() {
- @Override
- public void init(Properties properties) {}
-
- @Override
- public boolean addTimeSeries(int number) {
- // always return true, don't limit the number of series
- return true;
- }
-
- @Override
- public void deleteTimeSeries(int number) {
- // do nothing
- }
- };
+ // seriesNumerLimiter may be null, so we must check it before use it.
+ private ISeriesNumerMonitor seriesNumerMonitor = null;
// region MManager Singleton
private static class MManagerHolder {
@@ -235,8 +221,8 @@ public class MManager {
private static final MManager INSTANCE = new MManager();
}
- public void setSeriesNumerLimiter(ISeriesNumerLimiter seriesNumerLimiter) {
- this.seriesNumerLimiter = seriesNumerLimiter;
+ public void setSeriesNumerMonitor(ISeriesNumerMonitor seriesNumerMonitor) {
+ this.seriesNumerMonitor = seriesNumerMonitor;
}
/** we should not use this function in other place, but only in IoTDB class */
@@ -247,6 +233,18 @@ public class MManager {
// region Interfaces and Implementation of MManager initialization、snapshot、recover and clear
protected MManager() {
+ // init seriesNumerLimiter if there is.
+ // each mmanager instance will generate an ISeriesNumerLimiter instance
+ // So, if you want to share the ISeriesNumerLimiter instance, pls change this part of code.
+ ServiceLoader<ISeriesNumerMonitor> limiterServiceLoader =
+ ServiceLoader.load(ISeriesNumerMonitor.class);
+ for (ISeriesNumerMonitor loader : limiterServiceLoader) {
+ if (this.seriesNumerMonitor != null) {
+ // it means there is more than one ISeriesNumerLimiter implementation.
+ logger.warn("There are more than one ISeriesNumerLimiter implementation. pls check.");
+ }
+ this.seriesNumerMonitor = loader;
+ }
mtreeSnapshotInterval = config.getMtreeSnapshotInterval();
mtreeSnapshotThresholdTime = config.getMtreeSnapshotThresholdTime() * 1000L;
String schemaDir = config.getSchemaDir();
@@ -605,7 +603,7 @@ public class MManager {
+ "please increase MAX_HEAP_SIZE in iotdb-env.sh/bat and restart");
}
- if (!seriesNumerLimiter.addTimeSeries(1)) {
+ if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(1)) {
throw new SeriesNumberOverflowException();
}
try {
@@ -643,7 +641,9 @@ public class MManager {
}
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(1);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(1);
+ }
throw t;
}
@@ -730,7 +730,7 @@ public class MManager {
}
int seriesCount = plan.getMeasurements().size();
- if (!seriesNumerLimiter.addTimeSeries(seriesCount)) {
+ if (seriesNumerMonitor != null && !seriesNumerMonitor.addTimeSeries(seriesCount)) {
throw new SeriesNumberOverflowException();
}
@@ -759,7 +759,9 @@ public class MManager {
mNodeCache.invalidate(prefixPath);
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(seriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(seriesCount);
+ }
throw t;
}
@@ -892,7 +894,9 @@ public class MManager {
node = node.getParent();
}
totalNormalSeriesNumber.addAndGet(-1);
- seriesNumerLimiter.deleteTimeSeries(1);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(1);
+ }
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber);
@@ -936,7 +940,9 @@ public class MManager {
mtree.getAllTimeseriesCount(
storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false);
totalNormalSeriesNumber.addAndGet(-timeSeriesCount);
- seriesNumerLimiter.deleteTimeSeries(timeSeriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(timeSeriesCount);
+ }
// clear cached MNode
if (!allowToCreateNewSeries
&& totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
@@ -2529,7 +2535,9 @@ public class MManager {
node.setUseTemplate(false);
int seriesCount = node.getUpperTemplate().getMeasurementsCount();
totalTemplateSeriesNumber.addAndGet(-seriesCount);
- seriesNumerLimiter.deleteTimeSeries(seriesCount);
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(seriesCount);
+ }
// clear caches within MManger
mNodeCache.invalidate(node);
@@ -2555,7 +2563,8 @@ public class MManager {
String.format("Path [%s] has not been set any template.", node.getFullPath()));
}
- if (!seriesNumerLimiter.addTimeSeries(template.getMeasurementsCount())) {
+ if (seriesNumerMonitor != null
+ && !seriesNumerMonitor.addTimeSeries(template.getMeasurementsCount())) {
throw new SeriesNumberOverflowException();
}
@@ -2581,7 +2590,9 @@ public class MManager {
mountedMNode.setUseTemplate(true);
} catch (Throwable t) {
// roll back
- seriesNumerLimiter.deleteTimeSeries(template.getMeasurementsCount());
+ if (seriesNumerMonitor != null) {
+ seriesNumerMonitor.deleteTimeSeries(template.getMeasurementsCount());
+ }
throw t;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 66ffc23ac4..0c9c9c7dd2 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -53,8 +53,6 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
-import static org.apache.iotdb.db.utils.JarLoaderUtil.loadExternLib;
-
public class IoTDB implements IoTDBMBean {
private static final Logger logger = LoggerFactory.getLogger(IoTDB.class);
@@ -81,8 +79,6 @@ public class IoTDB implements IoTDBMBean {
}
IoTDB daemon = IoTDB.getInstance();
- loadExternLib(config);
-
daemon.active();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java b/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
deleted file mode 100644
index 30d0c28f47..0000000000
--- a/server/src/main/java/org/apache/iotdb/db/utils/JarLoaderUtil.java
+++ /dev/null
@@ -1,156 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.utils;
-
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
-import org.apache.iotdb.db.metadata.MManager;
-import org.apache.iotdb.external.api.IPropertiesLoader;
-import org.apache.iotdb.external.api.ISeriesNumerLimiter;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-
-import org.apache.commons.io.FileUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.net.UnknownHostException;
-import java.nio.file.Path;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Properties;
-import java.util.ServiceLoader;
-
-public class JarLoaderUtil {
-
- private static final Logger logger = LoggerFactory.getLogger(JarLoaderUtil.class);
-
- public static URL[] getExternalJarURLs(String jarDir) throws IOException {
- HashSet<File> fileSet =
- new HashSet<>(
- FileUtils.listFiles(
- SystemFileFactory.INSTANCE.getFile(jarDir), new String[] {"jar"}, true));
- return FileUtils.toURLs(fileSet.toArray(new File[0]));
- }
-
- public static void loadExternLib(IoTDBConfig config) {
- // load external properties
- String loaderDir = config.getExternalPropertiesLoaderDir();
-
- if (!(new File(loaderDir).exists())) {
- return;
- }
-
- Path externalPropertiesFile = IoTDBDescriptor.getInstance().getExternalPropsPath();
- URL[] loaderJarURLs;
- List<Properties> externalPropertiesList = new ArrayList<>();
- try {
- loaderJarURLs = getExternalJarURLs(loaderDir);
-
- if (loaderJarURLs == null || loaderJarURLs.length == 0) {
- return;
- }
-
- ClassLoader classLoader = new URLClassLoader(loaderJarURLs);
-
- // Use SPI to get all plugins' class
- ServiceLoader<IPropertiesLoader> loaders =
- ServiceLoader.load(IPropertiesLoader.class, classLoader);
-
- for (IPropertiesLoader loader : loaders) {
- if (loader == null) {
- logger.error("IPropertiesLoader(), loader is null.");
- continue;
- }
- Properties properties = loader.loadProperties(externalPropertiesFile.toAbsolutePath());
- if (properties != null) {
- externalPropertiesList.add(properties);
- }
- }
- } catch (Throwable t) {
- logger.error("error happened while loading external loader. ", t);
- // ignore
- }
-
- if (externalPropertiesList.size() != 1) {
- return;
- }
-
- // overwrite the default properties;
- for (Properties properties : externalPropertiesList) {
- try {
- IoTDBDescriptor.getInstance().loadProperties(properties);
- } catch (UnknownHostException e) {
- // ignore
- logger.error("error happened while loading external properties. ", e);
- }
- TSFileDescriptor.getInstance()
- .overwriteConfigByCustomSettings(TSFileDescriptor.getInstance().getConfig(), properties);
- }
-
- String limiterDir = config.getExternalLimiterDir();
-
- if (!(new File(loaderDir).exists())) {
- return;
- }
-
- URL[] limiterJarURLs;
-
- List<ISeriesNumerLimiter> limiterList = new ArrayList<>();
-
- try {
- limiterJarURLs = getExternalJarURLs(limiterDir);
-
- if (limiterJarURLs == null || limiterJarURLs.length == 0) {
- return;
- }
-
- ClassLoader classLoader = new URLClassLoader(limiterJarURLs);
-
- // Use SPI to get all plugins' class
- ServiceLoader<ISeriesNumerLimiter> limiters =
- ServiceLoader.load(ISeriesNumerLimiter.class, classLoader);
-
- for (ISeriesNumerLimiter limiter : limiters) {
- if (limiter == null) {
- logger.error("ISeriesNumerLimiter(), limiter is null.");
- continue;
- }
- for (Properties properties : externalPropertiesList) {
- limiter.init(properties);
- }
- limiterList.add(limiter);
- }
- } catch (Throwable t) {
- // ignore
- logger.error("error happened while loading external limiter. ", t);
- }
-
- if (limiterList.size() != 1) {
- return;
- }
-
- MManager.getInstance().setSeriesNumerLimiter(limiterList.get(0));
- }
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 060aba9189..df5df28446 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.tsfile.fileSystem.FSType;
import java.io.Serializable;
import java.nio.charset.Charset;
+import java.util.Properties;
/** TSFileConfig is a configuration class. Every variable is public and has default value. */
public class TSFileConfig implements Serializable {
@@ -145,6 +146,8 @@ public class TSFileConfig implements Serializable {
private double bloomFilterErrorRate = 0.05;
/** The amount of data iterate each time */
private int batchSize = 1000;
+ /** customizedProperties, this should be empty by default. */
+ private Properties customizedProperties = new Properties();
public TSFileConfig() {}
@@ -411,4 +414,12 @@ public class TSFileConfig implements Serializable {
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
+
+ public Properties getCustomizedProperties() {
+ return customizedProperties;
+ }
+
+ public void setCustomizedProperties(Properties customizedProperties) {
+ this.customizedProperties = customizedProperties;
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
index 711c5a4fd2..765bea8684 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileDescriptor.java
@@ -57,11 +57,11 @@ public class TSFileDescriptor {
private void init() {
Properties properties = loadProperties();
if (properties != null) {
- overwriteConfigByCustomSettings(this.conf, properties);
+ overwriteConfigByCustomSettings(properties);
}
}
- public void overwriteConfigByCustomSettings(TSFileConfig conf, Properties properties) {
+ public void overwriteConfigByCustomSettings(Properties properties) {
PropertiesOverWriter writer = new PropertiesOverWriter(properties);
writer.setInt(conf::setGroupSizeInByte, "group_size_in_byte");