You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/07/10 08:18:08 UTC
[iotdb] branch memory_allo_11 created (now c787e40979f)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch memory_allo_11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at c787e40979f Fix storage engine memory config initialization (#10077) (#10092)
This branch includes the following new commits:
new c787e40979f Fix storage engine memory config initialization (#10077) (#10092)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Fix storage engine memory config initialization (#10077) (#10092)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch memory_allo_11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c787e40979f3372dce76a4a7e2a242e4b23597bf
Author: Haonan <hh...@outlook.com>
AuthorDate: Fri Jun 9 11:45:28 2023 +0800
Fix storage engine memory config initialization (#10077) (#10092)
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 83 ++++++++++++++--------
2 files changed, 56 insertions(+), 32 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 617289e4dca..97ba72eed29 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -151,7 +151,7 @@ public class IoTDBConfig {
private double rejectProportion = 0.8;
/** The proportion of write memory for memtable */
- private double writeProportionForMemtable = 0.8;
+ private double writeProportionForMemtable = 0.76;
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
@@ -532,7 +532,8 @@ public class IoTDBConfig {
private long allocateMemoryForTimeIndex = allocateMemoryForRead * 200 / 1001;
/** Memory allocated proportion for time partition info */
- private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 50 / 1001;
+ private long allocateMemoryForTimePartitionInfo = allocateMemoryForStorageEngine * 8 / 10 / 20;
+
/**
* If true, we will estimate each query's possible memory footprint before executing it and deny
* it if its estimated memory exceeds current free memory
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 5b9fbd0ac5a..b9e1edc099f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1700,36 +1700,59 @@ public class IoTDBDescriptor {
}
private void initStorageEngineAllocate(Properties properties) {
- String allocationRatio = properties.getProperty("storage_engine_memory_proportion", "8:2");
- String[] proportions = allocationRatio.split(":");
- int proportionForWrite = Integer.parseInt(proportions[0].trim());
- int proportionForCompaction = Integer.parseInt(proportions[1].trim());
-
- double writeProportion =
- ((double) (proportionForWrite) / (double) (proportionForCompaction + proportionForWrite));
-
- String allocationRatioForWrite = properties.getProperty("write_memory_proportion", "19:1");
- proportions = allocationRatioForWrite.split(":");
- int proportionForMemTable = Integer.parseInt(proportions[0].trim());
- int proportionForTimePartitionInfo = Integer.parseInt(proportions[1].trim());
-
- double memtableProportionForWrite =
- ((double) (proportionForMemTable)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
-
- double timePartitionInfoForWrite =
- ((double) (proportionForTimePartitionInfo)
- / (double) (proportionForMemTable + proportionForTimePartitionInfo));
- conf.setWriteProportionForMemtable(writeProportion * memtableProportionForWrite);
-
- conf.setAllocateMemoryForTimePartitionInfo(
- (long)
- ((writeProportion * timePartitionInfoForWrite)
- * conf.getAllocateMemoryForStorageEngine()));
-
- conf.setCompactionProportion(
- ((double) (proportionForCompaction)
- / (double) (proportionForCompaction + proportionForWrite)));
+ long storageMemoryTotal = conf.getAllocateMemoryForStorageEngine();
+
+ int proportionSum = 10;
+ int writeProportion = 8;
+ int compactionProportion = 2;
+ int writeProportionSum = 20;
+ int memTableProportion = 19;
+ int timePartitionInfo = 1;
+
+ String storageMemoryAllocatePortion =
+ properties.getProperty("storage_engine_memory_proportion");
+ if (storageMemoryAllocatePortion != null) {
+ String[] proportions = storageMemoryAllocatePortion.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ proportionSum = loadedProportionSum;
+ writeProportion = Integer.parseInt(proportions[0].trim());
+ compactionProportion = Integer.parseInt(proportions[1].trim());
+ }
+ conf.setCompactionProportion((double) compactionProportion / (double) proportionSum);
+ }
+
+ String allocationRatioForWrite = properties.getProperty("write_memory_proportion");
+ if (allocationRatioForWrite != null) {
+ String[] proportions = allocationRatioForWrite.split(":");
+ int loadedProportionSum = 0;
+ for (String proportion : proportions) {
+ loadedProportionSum += Integer.parseInt(proportion.trim());
+ }
+
+ if (loadedProportionSum != 0) {
+ writeProportionSum = loadedProportionSum;
+ memTableProportion = Integer.parseInt(proportions[0].trim());
+ timePartitionInfo = Integer.parseInt(proportions[1].trim());
+ }
+ // memtableProportionForWrite = 19/20 default
+ double memtableProportionForWrite =
+ ((double) memTableProportion / (double) writeProportionSum);
+
+ // timePartitionInfoForWrite = 1/20 default
+ double timePartitionInfoForWrite = ((double) timePartitionInfo / (double) writeProportionSum);
+ // proportionForWrite = 8/10 default
+ double proportionForWrite = ((double) (writeProportion) / (double) proportionSum);
+ // writeProportionForMemtable = 8/10 * 19/20 = 0.76 default
+ conf.setWriteProportionForMemtable(proportionForWrite * memtableProportionForWrite);
+ // allocateMemoryForTimePartitionInfo = storageMemoryTotal * 8/10 * 1/20 default
+ conf.setAllocateMemoryForTimePartitionInfo(
+ (long) ((proportionForWrite * timePartitionInfoForWrite) * storageMemoryTotal));
+ }
}
private void initSchemaMemoryAllocate(Properties properties) {