You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/06/09 01:28:22 UTC
[iotdb] 01/01: Fix storage engine memory config initialization (#10077)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch memory_allo_12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d0140d3819f799ba56781c38594b9409ac8f0335
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Jun 8 19:35:32 2023 +0800
Fix storage engine memory config initialization (#10077)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 83 ++++++++++++++--------
2 files changed, 55 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b640f9198a2..2d147943b87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -154,7 +154,7 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;
/** The proportion of write memory for memtable */
- private double writeProportionForMemtable = 0.8;
+ private double writeProportionForMemtable = 0.76;
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
@@ -554,7 +554,7 @@ public class IoTDBConfig {
private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001;
/** Memory allocated proportion for time partition info */
- private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001;
+ private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 8 / 10 / 20;
/** Memory allocated proportion for wal pipe cache */
private long allocateMemoryForWALPipeCache = allocateMemoryForConsensus / 10;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 386920dbf4c..2f1c9fdee4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1767,36 +1767,59 @@ public class IoTDBDescriptor {
}
private void initStorageEngineAllocate(Properties properties) {
- String allocationRatio = properties.getProperty("storage_engine_memory_proportion", "8:2");
- String[] proportions = allocationRatio.split(":");
- int proportionForWrite = Integer.parseInt(proportions[0].trim());
- int proportionForCompaction = Integer.parseInt(proportions[1].trim());
-
- double writeProportion =
- ((double) (proportionForWrite) / (double) (proportionForCompaction + proportionForWrite));
-
- String allocationRatioForWrite = properties.getProperty("write_memory_proportion", "19:1");
- proportions = allocationRatioForWrite.split(":");
- int proportionForMemTable = Integer.parseInt(proportions[0].trim());
- int proportionForTimePartitionInfo = Integer.parseInt(proportions[1].trim());
-
- double memtableProportionForWrite =
- ((double) (proportionForMemTable)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
-
- double timePartitionInfoForWrite =
- ((double) (proportionForTimePartitionInfo)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
- conf.setWriteProportionForMemtable(writeProportion * memtableProportionForWrite);
-
- conf.setAllocateMemoryForTimePartitionInfo(
- (long)
- ((writeProportion * timePartitionInfoForWrite)
- * conf.getAllocateMemoryForStorageEngine()));
-
- conf.setCompactionProportion(
- ((double) (proportionForCompaction)
- / (double) (proportionForCompaction + proportionForWrite)));
+ long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine();
+
+ int proportionSum = 10;
+ int writeProportion = 8;
+ int compactionProportion = 2;
+ int writeProportionSum = 20;
+ int memTableProportion = 19;
+ int timePartitionInfo = 1;
+
+ String storageMemoryAllocatePortion =
+ properties.getProperty("storage_engine_memory_proportion");
+ if (storageMemoryAllocatePortion != null) {
+ String[] proportions = storageMemoryAllocatePortion.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ proportionSum = loadedProportionSum;
+ writeProportion = Integer.parseInt(proportions[0].trim());
+ compactionProportion = Integer.parseInt(proportions[1].trim());
+ }
+ conf.setCompactionProportion((double) compactionProportion / (double) proportionSum);
+ }
+
+ String allocationRatioForWrite = properties.getProperty("write_memory_proportion");
+ if (allocationRatioForWrite != null) {
+ String[] proportions = allocationRatioForWrite.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ writeProportionSum = loadedProportionSum;
+ memTableProportion = Integer.parseInt(proportions[0].trim());
+ timePartitionInfo = Integer.parseInt(proportions[1].trim());
+ }
+ // memtableProportionForWrite = 19/20 default
+ double memtableProportionForWrite =
+ ((double) memTableProportion / (double) writeProportionSum);
+
+ // timePartitionInfoForWrite = 1/20 default
+ double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum);
+ // proportionForWrite = 8/10 default
+ double proportionForWrite = ((double) (writeProportion) / (double) proportionSum);
+ // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
+ conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite);
+ // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default
+ conf.setAllocateMemoryForTimePartitionInfo(
+ (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal));
+ }
}
private void initSchemaMemoryAllocate(Properties properties) {