You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2020/06/15 13:03:27 UTC
[incubator-iotdb] 01/02: init
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch OOMImprove
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 9b904d07ae44a8ce1a9633b3b942192416e92bb7
Author: JackieTien97 <Ja...@foxmail.com>
AuthorDate: Mon Jun 15 17:45:31 2020 +0800
init
---
.../assembly/resources/conf/iotdb-engine.properties | 4 ++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 +++++++++++-------
.../org/apache/iotdb/db/service/TSServiceImpl.java | 14 +++++++++++---
4 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 846228c..abb6fd9 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -226,6 +226,10 @@ write_read_free_memory_proportion=6:3:1
# primitive array size (length of each array) in array pool
primitive_array_size=128
+# allowed max numbers of deduplicated path in one query
+# it's just an advised value, the real limitation will be the smaller one between this and the one we calculated
+max_deduplicated_path_num=10000
+
####################
### Upgrade Configurations
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 23c111a..774e456 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -128,6 +128,8 @@ public class IoTDBConfig {
*/
private long allocateMemoryForRead = Runtime.getRuntime().maxMemory() * 3 / 10;
+ private long allocateMemoryForReadWithoutCache = Runtime.getRuntime().maxMemory() * 21 / 100;
+
/**
* Is dynamic parameter adapter enable.
*/
@@ -1086,6 +1088,14 @@ public class IoTDBConfig {
this.allocateMemoryForRead = allocateMemoryForRead;
}
+ public long getAllocateMemoryForReadWithoutCache() {
+ return allocateMemoryForReadWithoutCache;
+ }
+
+ public void setAllocateMemoryForReadWithoutCache(long allocateMemoryForReadWithoutCache) {
+ this.allocateMemoryForReadWithoutCache = allocateMemoryForReadWithoutCache;
+ }
+
public boolean isEnableExternalSort() {
return enableExternalSort;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 333b2d8..5db69b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -428,7 +428,6 @@ public class IoTDBDescriptor {
//if using org.apache.iotdb.db.auth.authorizer.OpenIdAuthorizer, openID_url is needed.
conf.setOpenIdProviderUrl(properties.getProperty("openID_url", ""));
-
// At the same time, set TSFileConfig
TSFileDescriptor.getInstance().getConfig()
.setTSFileStorageFs(FSType.valueOf(
@@ -493,12 +492,15 @@ public class IoTDBDescriptor {
conf.setAutoCreateSchemaEnabled(
Boolean.parseBoolean(properties.getProperty("enable_auto_create_schema",
Boolean.toString(conf.isAutoCreateSchemaEnabled()).trim())));
- conf.setBooleanStringInferType(TSDataType.valueOf(properties.getProperty("boolean_string_infer_type",
- conf.getBooleanStringInferType().toString())));
- conf.setIntegerStringInferType(TSDataType.valueOf(properties.getProperty("integer_string_infer_type",
- conf.getIntegerStringInferType().toString())));
- conf.setFloatingStringInferType(TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
- conf.getFloatingStringInferType().toString())));
+ conf.setBooleanStringInferType(
+ TSDataType.valueOf(properties.getProperty("boolean_string_infer_type",
+ conf.getBooleanStringInferType().toString())));
+ conf.setIntegerStringInferType(
+ TSDataType.valueOf(properties.getProperty("integer_string_infer_type",
+ conf.getIntegerStringInferType().toString())));
+ conf.setFloatingStringInferType(
+ TSDataType.valueOf(properties.getProperty("floating_string_infer_type",
+ conf.getFloatingStringInferType().toString())));
conf.setNanStringInferType(TSDataType.valueOf(properties.getProperty("nan_string_infer_type",
conf.getNanStringInferType().toString())));
conf.setDefaultStorageGroupLevel(
@@ -676,6 +678,8 @@ public class IoTDBDescriptor {
maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum);
conf.setAllocateMemoryForTimeSeriesMetaDataCache(
maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum);
+ conf.setAllocateMemoryForReadWithoutCache(
+ maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum);
} catch (Exception e) {
throw new RuntimeException(
"Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion"
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index d17457e..fd9f086 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -138,6 +138,9 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
// The statementId is unique in one IoTDB instance.
private AtomicLong statementIdGenerator = new AtomicLong();
+ // current total free memory for reading process(not including the cache memory)
+ private final AtomicLong totalFreeMemoryForRead;
+
// (sessionId -> Set(statementId))
private Map<Long, Set<Long>> sessionId2StatementId = new ConcurrentHashMap<>();
// (statementId -> Set(queryId))
@@ -154,6 +157,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
public TSServiceImpl() throws QueryProcessException {
processor = new Planner();
executor = new PlanExecutor();
+ totalFreeMemoryForRead = new AtomicLong(
+ IoTDBDescriptor.getInstance().getConfig().getAllocateMemoryForReadWithoutCache());
}
public static List<SqlArgument> getSqlArgumentList() {
@@ -420,7 +425,8 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
} catch (SQLParserException e) {
logger.error("Error occurred when executing {}, check metadata error: ", statement, e);
result.add(RpcUtils.getStatus(
- TSStatusCode.SQL_PARSE_ERROR, ERROR_PARSING_SQL + " " + statement + " " + e.getMessage()));
+ TSStatusCode.SQL_PARSE_ERROR,
+ ERROR_PARSING_SQL + " " + statement + " " + e.getMessage()));
return false;
} catch (QueryProcessException e) {
logger.info(
@@ -687,8 +693,10 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
// Last Query should return different respond instead of the static one
// because the query dataset and query id is different although the header of last query is same.
return StaticResps.LAST_RESP.deepCopy();
- } else if (plan instanceof AggregationPlan && ((AggregationPlan)plan).getLevel() >= 0) {
- Map<String, Long> finalPaths = FilePathUtils.getPathByLevel(((AggregationPlan)plan).getDeduplicatedPaths(), ((AggregationPlan)plan).getLevel(), null);
+ } else if (plan instanceof AggregationPlan && ((AggregationPlan) plan).getLevel() >= 0) {
+ Map<String, Long> finalPaths = FilePathUtils
+ .getPathByLevel(((AggregationPlan) plan).getDeduplicatedPaths(),
+ ((AggregationPlan) plan).getLevel(), null);
for (Map.Entry<String, Long> entry : finalPaths.entrySet()) {
respColumns.add("count(" + entry.getKey() + ")");
columnsTypes.add(TSDataType.INT64.toString());