You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/08/11 02:20:17 UTC
[iotdb] branch MemoryControl updated: Add enable_query_memory_estimation and merge time_index_memory_proportion into chunk_timeseriesmeta_free_memory_proportion
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch MemoryControl
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/MemoryControl by this push:
new 4b09715779 Add enable_query_memory_estimation and merge time_index_memory_proportion into chunk_timeseriesmeta_free_memory_proportion
4b09715779 is described below
commit 4b09715779683e8848fac65ae0b294a4c27a1fbe
Author: JackieTien97 <ja...@gmail.com>
AuthorDate: Thu Aug 11 10:20:09 2022 +0800
Add enable_query_memory_estimation and merge time_index_memory_proportion into chunk_timeseriesmeta_free_memory_proportion
---
.../integration/IoTDBManageTsFileResourceIT.java | 7 ++--
.../resources/conf/iotdb-datanode.properties | 14 ++++----
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 +++++++++++++++-------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 39 ++++++++++++++++-----
.../db/mpp/plan/planner/LocalExecutionPlanner.java | 6 ++++
.../iotdb/db/rescon/TsFileResourceManager.java | 3 +-
.../iotdb/db/rescon/ResourceManagerTest.java | 10 ++----
7 files changed, 76 insertions(+), 43 deletions(-)
diff --git a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBManageTsFileResourceIT.java b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBManageTsFileResourceIT.java
index 41a270e637..96031214e7 100644
--- a/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBManageTsFileResourceIT.java
+++ b/integration/src/test/java/org/apache/iotdb/db/integration/IoTDBManageTsFileResourceIT.java
@@ -54,8 +54,7 @@ import static org.junit.Assert.assertTrue;
public class IoTDBManageTsFileResourceIT {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
private TsFileResourceManager tsFileResourceManager = TsFileResourceManager.getInstance();
- private double prevTimeIndexMemoryProportion;
- private double prevTimeIndexMemoryThreshold;
+ private long prevTimeIndexMemoryThreshold;
private int prevCompactionThreadNum;
private static String[] unSeqSQLs =
@@ -89,7 +88,7 @@ public class IoTDBManageTsFileResourceIT {
@Before
public void setUp() throws ClassNotFoundException {
EnvironmentUtils.envSetUp();
- prevTimeIndexMemoryProportion = CONFIG.getTimeIndexMemoryProportion();
+ prevTimeIndexMemoryThreshold = CONFIG.getAllocateMemoryForTimeIndex();
prevCompactionThreadNum = CONFIG.getConcurrentCompactionThread();
Class.forName(Config.JDBC_DRIVER_NAME);
}
@@ -97,8 +96,6 @@ public class IoTDBManageTsFileResourceIT {
@After
public void tearDown() throws Exception {
EnvironmentUtils.cleanEnv();
- prevTimeIndexMemoryThreshold =
- prevTimeIndexMemoryProportion * CONFIG.getAllocateMemoryForRead();
tsFileResourceManager.setTimeIndexMemoryThreshold(prevTimeIndexMemoryThreshold);
CONFIG.setConcurrentCompactionThread(prevCompactionThreadNum);
}
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 35a0f89d14..ed663c4755 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -439,10 +439,6 @@ timestamp_precision=ms
# Datatype: double
# flush_proportion=0.4
-# Ratio of read memory allocated for timeIndex, 0.2 by default
-# Datatype: double
-# time_index_memory_proportion=0.2
-
# Ratio of write memory allocated for buffered arrays, 0.6 by default
# Datatype: double
# buffered_arrays_memory_proportion=0.6
@@ -476,6 +472,10 @@ timestamp_precision=ms
# Datatype: int
# io_task_queue_size_for_flushing=10
+# If true, we will estimate each query's possible memory footprint before executing it and deny it if its estimated memory exceeds current free memory
+# Datatype: bool
+# enable_query_memory_estimation=true
+
####################
### Upgrade Configurations
####################
@@ -637,9 +637,9 @@ timestamp_precision=ms
# Datatype: boolean
# meta_data_cache_enable=true
-# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange.
-# The parameter form is a:b:c:d:e:f, where a, b, c, d, e and f are integers. for example: 1:1:1:1:1:1 , 1:100:200:50:300:350
-# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:300:350
+# Read memory Allocation Ratio: BloomFilterCache : ChunkCache : TimeSeriesMetadataCache : Coordinator : Operators : DataExchange : timeIndex in TsFileResourceList : others.
+# The parameter form is a:b:c:d:e:f:g:h, where a, b, c, d, e, f, g and h are integers. for example: 1:1:1:1:1:1:1:1 , 1:100:200:50:200:200:200:50
+# chunk_timeseriesmeta_free_memory_proportion=1:100:200:50:200:200:200:50
####################
### LAST Cache Configuration
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 17234c37be..1f387be3db 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -137,9 +137,6 @@ public class IoTDBConfig {
/** Ratio of memory allocated for buffered arrays */
private double bufferedArraysMemoryProportion = 0.6;
- /** Memory allocated proportion for timeIndex */
- private double timeIndexMemoryProportion = 0.2;
-
/** Flush proportion for system */
private double flushProportion = 0.4;
@@ -475,10 +472,19 @@ public class IoTDBConfig {
private long allocateMemoryForCoordinator = allocateMemoryForRead * 50 / 1001;
/** Memory allocated for operators */
- private long allocateMemoryForOperators = allocateMemoryForRead * 300 / 1001;
+ private long allocateMemoryForOperators = allocateMemoryForRead * 200 / 1001;
/** Memory allocated for operators */
- private long allocateMemoryForDataExchange = allocateMemoryForRead * 350 / 1001;
+ private long allocateMemoryForDataExchange = allocateMemoryForRead * 200 / 1001;
+
+ /** Memory allocated proportion for timeIndex */
+ private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001;
+
+ /**
+ * If true, we will estimate each query's possible memory footprint before executing it and deny
+ * it if its estimated memory exceeds current free memory
+ */
+ private boolean enableQueryMemoryEstimation = true;
/** Whether to enable Last cache */
private boolean lastCacheEnable = true;
@@ -1665,14 +1671,6 @@ public class IoTDBConfig {
this.bufferedArraysMemoryProportion = bufferedArraysMemoryProportion;
}
- public double getTimeIndexMemoryProportion() {
- return timeIndexMemoryProportion;
- }
-
- public void setTimeIndexMemoryProportion(double timeIndexMemoryProportion) {
- this.timeIndexMemoryProportion = timeIndexMemoryProportion;
- }
-
public double getFlushProportion() {
return flushProportion;
}
@@ -1956,6 +1954,22 @@ public class IoTDBConfig {
this.allocateMemoryForDataExchange = allocateMemoryForDataExchange;
}
+ public long getAllocateMemoryForTimeIndex() {
+ return allocateMemoryForTimeIndex;
+ }
+
+ public void setAllocateMemoryForTimeIndex(long allocateMemoryForTimeIndex) {
+ this.allocateMemoryForTimeIndex = allocateMemoryForTimeIndex;
+ }
+
+ public boolean isEnableQueryMemoryEstimation() {
+ return enableQueryMemoryEstimation;
+ }
+
+ public void setEnableQueryMemoryEstimation(boolean enableQueryMemoryEstimation) {
+ this.enableQueryMemoryEstimation = enableQueryMemoryEstimation;
+ }
+
public boolean isLastCacheEnabled() {
return lastCacheEnable;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index e6c7825969..16adccd7db 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -200,12 +200,6 @@ public class IoTDBDescriptor {
"buffered_arrays_memory_proportion",
Double.toString(conf.getBufferedArraysMemoryProportion()))));
- conf.setTimeIndexMemoryProportion(
- Double.parseDouble(
- properties.getProperty(
- "time_index_memory_proportion",
- Double.toString(conf.getTimeIndexMemoryProportion()))));
-
conf.setFlushProportion(
Double.parseDouble(
properties.getProperty(
@@ -1416,6 +1410,13 @@ public class IoTDBDescriptor {
.trim()));
conf.setIpWhiteList(properties.getProperty("ip_white_list", conf.getIpWhiteList()));
+ // update enable query memory estimation for memory control
+ conf.setEnableQueryMemoryEstimation(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_query_memory_estimation",
+ Boolean.toString(conf.isEnableQueryMemoryEstimation()))));
+
// update wal config
long prevDeleteWalFilesPeriodInMs = conf.getDeleteWalFilesPeriodInMs();
loadWALHotModifiedProps(properties);
@@ -1480,9 +1481,11 @@ public class IoTDBDescriptor {
"max_deduplicated_path_num",
Integer.toString(conf.getMaxQueryDeduplicatedPathNum()))));
- if (!conf.isMetaDataCacheEnable()) {
- return;
- }
+ conf.setEnableQueryMemoryEstimation(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_query_memory_estimation",
+ Boolean.toString(conf.isEnableQueryMemoryEstimation()))));
String queryMemoryAllocateProportion =
properties.getProperty("chunk_timeseriesmeta_free_memory_proportion");
@@ -1507,6 +1510,8 @@ public class IoTDBDescriptor {
maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum);
conf.setAllocateMemoryForDataExchange(
maxMemoryAvailable * Integer.parseInt(proportions[5].trim()) / proportionSum);
+ conf.setAllocateMemoryForTimeIndex(
+ maxMemoryAvailable * Integer.parseInt(proportions[6].trim()) / proportionSum);
} catch (Exception e) {
throw new RuntimeException(
"Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion"
@@ -1515,6 +1520,22 @@ public class IoTDBDescriptor {
}
}
}
+
+ // metadata cache is disabled, we need to move all their allocated memory to other parts
+ if (!conf.isMetaDataCacheEnable()) {
+ long sum =
+ conf.getAllocateMemoryForBloomFilterCache()
+ + conf.getAllocateMemoryForChunkCache()
+ + conf.getAllocateMemoryForTimeSeriesMetaDataCache();
+ conf.setAllocateMemoryForBloomFilterCache(0);
+ conf.setAllocateMemoryForChunkCache(0);
+ conf.setAllocateMemoryForTimeSeriesMetaDataCache(0);
+ long partForDataExchange = sum / 2;
+ long partForOperators = sum - partForDataExchange;
+ conf.setAllocateMemoryForDataExchange(
+ conf.getAllocateMemoryForDataExchange() + partForDataExchange);
+ conf.setAllocateMemoryForOperators(conf.getAllocateMemoryForOperators() + partForOperators);
+ }
}
private void initSchemaMemoryAllocate(Properties properties) {
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
index 451cc208d1..51056ba3e8 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/LocalExecutionPlanner.java
@@ -110,6 +110,12 @@ public class LocalExecutionPlanner {
private void checkMemory(Operator root, FragmentInstanceStateMachine stateMachine)
throws MemoryNotEnoughException {
+
+ // if it is disabled, just return
+ if (!IoTDBDescriptor.getInstance().getConfig().isEnableQueryMemoryEstimation()) {
+ return;
+ }
+
long estimatedMemorySize = root.calculateMaxPeekMemory();
synchronized (this) {
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TsFileResourceManager.java b/server/src/main/java/org/apache/iotdb/db/rescon/TsFileResourceManager.java
index ae6ab41567..b20e84216e 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TsFileResourceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/TsFileResourceManager.java
@@ -35,8 +35,7 @@ public class TsFileResourceManager {
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
/** threshold total memory for all TimeIndex */
- private double TIME_INDEX_MEMORY_THRESHOLD =
- CONFIG.getAllocateMemoryForRead() * CONFIG.getTimeIndexMemoryProportion();
+ private double TIME_INDEX_MEMORY_THRESHOLD = CONFIG.getAllocateMemoryForTimeIndex();
/** store the sealed TsFileResource, sorted by priority of TimeIndex */
private final TreeSet<TsFileResource> sealedTsFileResources =
diff --git a/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java b/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java
index 15da188fa1..4eb277d663 100644
--- a/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/rescon/ResourceManagerTest.java
@@ -74,15 +74,14 @@ public class ResourceManagerTest {
List<TsFileResource> unseqResources = new ArrayList<>();
private static final IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
- private TsFileResourceManager tsFileResourceManager = TsFileResourceManager.getInstance();
- private double prevTimeIndexMemoryProportion;
- private double prevTimeIndexMemoryThreshold;
+ private final TsFileResourceManager tsFileResourceManager = TsFileResourceManager.getInstance();
+ private long prevTimeIndexMemoryThreshold;
private TimeIndexLevel timeIndexLevel;
@Before
public void setUp() throws IOException, WriteProcessException, MetadataException {
IoTDB.configManager.init();
- prevTimeIndexMemoryProportion = CONFIG.getTimeIndexMemoryProportion();
+ prevTimeIndexMemoryThreshold = CONFIG.getAllocateMemoryForTimeIndex();
timeIndexLevel = CONFIG.getTimeIndexLevel();
prepareSeries();
}
@@ -92,10 +91,7 @@ public class ResourceManagerTest {
removeFiles();
seqResources.clear();
unseqResources.clear();
- CONFIG.setTimeIndexMemoryProportion(prevTimeIndexMemoryProportion);
CONFIG.setTimeIndexLevel(String.valueOf(timeIndexLevel));
- prevTimeIndexMemoryThreshold =
- prevTimeIndexMemoryProportion * CONFIG.getAllocateMemoryForRead();
tsFileResourceManager.setTimeIndexMemoryThreshold(prevTimeIndexMemoryThreshold);
ChunkCache.getInstance().clear();
TimeSeriesMetadataCache.getInstance().clear();