You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/06/09 01:28:21 UTC
[iotdb] branch memory_allo_12 created (now d0140d3819f)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch memory_allo_12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at d0140d3819f Fix storage engine memory config initialization (#10077)
This branch includes the following new commits:
new d0140d3819f Fix storage engine memory config initialization (#10077)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Fix storage engine memory config initialization (#10077)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch memory_allo_12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d0140d3819f799ba56781c38594b9409ac8f0335
Author: Haonan <hh...@outlook.com>
AuthorDate: Thu Jun 8 19:35:32 2023 +0800
Fix storage engine memory config initialization (#10077)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 83 ++++++++++++++--------
2 files changed, 55 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b640f9198a2..2d147943b87 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -154,7 +154,7 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;
/** The proportion of write memory for memtable */
- private double writeProportionForMemtable = 0.8;
+ private double writeProportionForMemtable = 0.76;
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
@@ -554,7 +554,7 @@ public class IoTDBConfig {
private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001;
/** Memory allocated proportion for time partition info */
- private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001;
+ private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 8 / 10 / 20;
/** Memory allocated proportion for wal pipe cache */
private long allocateMemoryForWALPipeCache = allocateMemoryForConsensus / 10;
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 386920dbf4c..2f1c9fdee4c 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1767,36 +1767,59 @@ public class IoTDBDescriptor {
}
private void initStorageEngineAllocate(Properties properties) {
- String allocationRatio = properties.getProperty("storage_engine_memory_proportion", "8:2");
- String[] proportions = allocationRatio.split(":");
- int proportionForWrite = Integer.parseInt(proportions[0].trim());
- int proportionForCompaction = Integer.parseInt(proportions[1].trim());
-
- double writeProportion =
- ((double) (proportionForWrite) / (double) (proportionForCompaction + proportionForWrite));
-
- String allocationRatioForWrite = properties.getProperty("write_memory_proportion", "19:1");
- proportions = allocationRatioForWrite.split(":");
- int proportionForMemTable = Integer.parseInt(proportions[0].trim());
- int proportionForTimePartitionInfo = Integer.parseInt(proportions[1].trim());
-
- double memtableProportionForWrite =
- ((double) (proportionForMemTable)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
-
- double timePartitionInfoForWrite =
- ((double) (proportionForTimePartitionInfo)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
- conf.setWriteProportionForMemtable(writeProportion * memtableProportionForWrite);
-
- conf.setAllocateMemoryForTimePartitionInfo(
- (long)
- ((writeProportion * timePartitionInfoForWrite)
- * conf.getAllocateMemoryForStorageEngine()));
-
- conf.setCompactionProportion(
- ((double) (proportionForCompaction)
- / (double) (proportionForCompaction + proportionForWrite)));
+ long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine();
+
+ int proportionSum = 10;
+ int writeProportion = 8;
+ int compactionProportion = 2;
+ int writeProportionSum = 20;
+ int memTableProportion = 19;
+ int timePartitionInfo = 1;
+
+ String storageMemoryAllocatePortion =
+ properties.getProperty("storage_engine_memory_proportion");
+ if (storageMemoryAllocatePortion != null) {
+ String[] proportions = storageMemoryAllocatePortion.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ proportionSum = loadedProportionSum;
+ writeProportion = Integer.parseInt(proportions[0].trim());
+ compactionProportion = Integer.parseInt(proportions[1].trim());
+ }
+ conf.setCompactionProportion((double) compactionProportion / (double) proportionSum);
+ }
+
+ String allocationRatioForWrite = properties.getProperty("write_memory_proportion");
+ if (allocationRatioForWrite != null) {
+ String[] proportions = allocationRatioForWrite.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ writeProportionSum = loadedProportionSum;
+ memTableProportion = Integer.parseInt(proportions[0].trim());
+ timePartitionInfo = Integer.parseInt(proportions[1].trim());
+ }
+ // memtableProportionForWrite = 19/20 default
+ double memtableProportionForWrite =
+ ((double) memTableProportion / (double) writeProportionSum);
+
+ // timePartitionInfoForWrite = 1/20 default
+ double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum);
+ // proportionForWrite = 8/10 default
+ double proportionForWrite = ((double) (writeProportion) / (double) proportionSum);
+ // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
+ conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite);
+ // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default
+ conf.setAllocateMemoryForTimePartitionInfo(
+ (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal));
+ }
}
private void initSchemaMemoryAllocate(Properties properties) {