You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2025/02/27 03:35:37 UTC
(iotdb) branch feature/memory_transfer updated (51ed178f4a1 -> 7d7f77a8d79)
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/memory_transfer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 51ed178f4a1 Merge branch 'master' into feature/memory_transfer
add 14f172c6845 Merge branch 'master' into feature/memory_collect
add a864d9fabc4 Fix class name of RPCServiceMBean #14940
add bb8a5f74356 Pipe: Fix the protential resource leak issue of async client when async manager is closed (#14929)
add 72dd41d62ed Pipe: Add memory control mechanism for TsFile batch mode in sinks (#14911)
add 9733c7540cf Delete remove node logic in ServerCommandLine #14933
add ab2ab36fa7d Subscription: check if all topic messages have been consumed when client polling (#14937)
add 5d38f0c251e Fix the missing deletion when concurrent deletion and insertion compaction (#14955)
add dbe2d37c389 enhance wal compatibility (#14954)
add de10ea78911 [AINode] Optimize the error information when dropping built-in model (#14943)
add fa9711e1146 IoTConsensusV2: Use custom commitIndex to indicate synchronization progress instead of pipe's commitIndex (#14624)
add 29cc71b599f Add timestamp suffix to vulnerability check report. (#14960)
add fb2cb7a394b Fix state setting of AbstractOperatePipeProcedureV2 & Enhance state checking of StateMachineProcedure #14959
add 9e79d85b7a8 peformance regression of sequential inserting (#14951)
add 95972550d8c Introduce flushing on specific regions & IoTConsensusV2: Flush old leader when leader transfer & Delay the execution of invalidateSchemaCache after leader change (#14910)
add ecaac56bef9 Merge branch 'master' into feature/memory_collect
add 6f916c11850 move some manager to memory config
add 90e00789b67 move some manager to memory config(finish)
add f6c3c5da617 Adapt to Mybatis generator (#14936)
add 1e6513b1fc5 IoTConsensusV2: Multi-Dirs balance with sequence rolling strategy in receiver (#14962)
add 1a4ac0ea0ad Merge branch 'master' into feature/memory_collect
new 93f0aa26d8b merge feature/memory_collect
new 7d7f77a8d79 update format
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/vulnerability-check.yml | 7 +-
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 2 +
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +-
.../IoTDBRelationalDatabaseMetadata.java | 52 +++-
.../base/AbstractSubscriptionConsumer.java | 22 +-
.../iotdb/confignode/manager/ConfigManager.java | 9 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ModelManager.java | 5 +
.../manager/load/balancer/RouteBalancer.java | 152 ++++++---
.../iotdb/confignode/manager/node/NodeManager.java | 8 +
.../iotdb/confignode/persistence/ModelInfo.java | 2 +-
.../procedure/impl/StateMachineProcedure.java | 9 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 +
.../iotdb/confignode/service/ConfigNode.java | 44 +--
.../consensus/config/PipeConsensusConfig.java | 21 +-
.../consensus/pipe/PipeConsensusServerImpl.java | 21 +-
...xManager.java => ReplicateProgressManager.java} | 8 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 71 ++++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 286 +----------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 106 ++++---
.../db/consensus/DataRegionConsensusImpl.java | 10 +-
.../client/IoTDBDataNodeAsyncClientManager.java | 4 +
.../evolvable/batch/PipeTabletEventBatch.java | 37 ++-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 41 +--
.../batch/PipeTabletEventTsFileBatch.java | 11 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 9 +-
.../PipeConsensusTabletInsertionEventHandler.java | 8 +-
.../PipeConsensusTsFileInsertionEventHandler.java | 8 +-
.../PipeConsensusTransferBatchReqBuilder.java | 5 +-
.../async/IoTDBDataRegionAsyncConnector.java | 1 +
....java => ReplicateProgressDataNodeManager.java} | 36 ++-
.../deletion/DeletionResourceManager.java | 4 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 5 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 17 +-
.../event/realtime/PipeRealtimeEventFactory.java | 71 ++++-
...oricalDataRegionTsFileAndDeletionExtractor.java | 17 ++
.../realtime/assigner/PipeDataRegionAssigner.java | 2 +
.../listener/PipeInsertionDataNodeListener.java | 16 +-
.../pipeconsensus/PipeConsensusProcessor.java | 43 ++-
.../pipeconsensus/PipeConsensusReceiver.java | 338 +++++++++------------
.../db/pipe/resource/memory/PipeMemoryManager.java | 5 +-
.../execution/memory/LocalMemoryManager.java | 3 +-
.../analyze/cache/partition/PartitionCache.java | 4 +-
.../cache/schema/DataNodeDevicePathCache.java | 4 +-
.../plan/planner/LocalExecutionPlanner.java | 6 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 4 +-
.../metric/SchemaEngineCachedMetric.java | 7 +-
.../rescon/MemSchemaEngineStatistics.java | 5 +-
.../ReleaseFlushStrategySizeBasedImpl.java | 7 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 64 +---
.../iotdb/db/service/ExternalRPCService.java | 2 +-
...viceMBean.java => ExternalRPCServiceMBean.java} | 2 +-
.../metrics/memory/ConsensusMemoryMetrics.java | 6 +-
.../metrics/memory/GlobalMemoryMetrics.java | 6 +-
.../metrics/memory/OffHeapMemoryMetrics.java | 6 +-
.../metrics/memory/QueryEngineMemoryMetrics.java | 30 +-
.../metrics/memory/SchemaEngineMemoryMetrics.java | 16 +-
.../metrics/memory/StorageEngineMemoryMetrics.java | 28 +-
.../metrics/memory/StreamEngineMemoryMetrics.java | 6 +-
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../db/storageengine/buffer/BloomFilterCache.java | 4 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 6 +-
.../buffer/TimeSeriesMetadataCache.java | 4 +-
.../storageengine/dataregion/DataRegionInfo.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 61 +++-
.../memtable/AlignedWritableMemChunk.java | 13 +
.../memtable/AlignedWritableMemChunkGroup.java | 15 +-
.../dataregion/memtable/IMemTable.java | 2 +
.../memtable/IWritableMemChunkGroup.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 41 ++-
.../dataregion/memtable/WritableMemChunk.java | 8 +
.../dataregion/memtable/WritableMemChunkGroup.java | 20 ++
.../dataregion/tsfile/TsFileResource.java | 8 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../rescon/memory/PrimitiveArrayManager.java | 5 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 18 +-
.../rescon/memory/TimePartitionManager.java | 5 +-
.../rescon/memory/TsFileResourceManager.java | 6 +-
.../db/utils/datastructure/AlignedTVList.java | 44 ++-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 38 ++-
.../db/utils/datastructure/BooleanTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/FloatTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/IntTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/LongTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/TVList.java | 70 ++++-
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 10 +-
.../dataregion/memtable/TsFileProcessorTest.java | 36 +--
.../rescon/memory/ResourceManagerTest.java | 5 +-
.../rescon/memory/TimePartitionManagerTest.java | 7 +-
.../datastructure/PrimitiveArrayManagerTest.java | 5 +-
.../async/AsyncPipeDataTransferServiceClient.java | 14 +
.../commons/memory/AtomicLongMemoryBlock.java | 36 +--
.../apache/iotdb/commons/memory/IMemoryBlock.java | 18 +-
.../apache/iotdb/commons/memory/MemoryConfig.java | 315 +++++++++++++++++++
.../iotdb/commons/memory/MemoryException.java | 7 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 33 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 17 +-
.../iotdb/commons/memory/MemoryManagerTest.java | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/pipeconsensus.thrift | 5 +-
103 files changed, 1730 insertions(+), 1166 deletions(-)
rename iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/pipe/consensuspipe/{ProgressIndexManager.java => ReplicateProgressManager.java} (81%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/consensus/{ProgressIndexDataNodeManager.java => ReplicateProgressDataNodeManager.java} (78%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/{RPCServiceMBean.java => ExternalRPCServiceMBean.java} (96%)
create mode 100644 iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryConfig.java
(iotdb) 02/02: update format
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/memory_transfer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d7f77a8d79ccc5e332521fcf733ebf1b53b70ed
Author: spricoder <sp...@qq.com>
AuthorDate: Thu Feb 27 11:35:24 2025 +0800
update format
---
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 4fec25153ba..b632c34f274 100755
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -2222,17 +2222,18 @@ public class IoTDBDescriptor {
} else {
pipeMemorySize =
(memoryAvailable
- - (memoryConfig.getStorageEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getQueryEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getConsensusMemoryManager().getTotalMemorySizeInBytes()))
+ - (memoryConfig.getStorageEngineMemoryManager().getTotalMemorySizeInBytes()
+ + memoryConfig.getQueryEngineMemoryManager().getTotalMemorySizeInBytes()
+ + memoryConfig.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes()
+ + memoryConfig.getConsensusMemoryManager().getTotalMemorySizeInBytes()))
/ 2;
}
}
}
// on heap memory manager
MemoryManager onheapMemoryManager =
- MemoryConfig.global().getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().totalMemory());
+ MemoryConfig.global()
+ .getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().totalMemory());
memoryConfig.setOnHeapMemoryManager(onheapMemoryManager);
// storage engine memory manager
MemoryManager storageEngineMemoryManager =
(iotdb) 01/02: merge feature/memory_collect
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/memory_transfer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 93f0aa26d8bf9a9c85d51c7047b2df00ff32e66e
Merge: 51ed178f4a1 1a4ac0ea0ad
Author: spricoder <sp...@qq.com>
AuthorDate: Thu Feb 27 11:34:27 2025 +0800
merge feature/memory_collect
.github/workflows/vulnerability-check.yml | 7 +-
.../org/apache/iotdb/ainode/it/AINodeBasicIT.java | 2 +
.../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 25 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 31 +-
.../IoTDBRelationalDatabaseMetadata.java | 52 +++-
.../base/AbstractSubscriptionConsumer.java | 22 +-
.../iotdb/confignode/manager/ConfigManager.java | 9 +
.../apache/iotdb/confignode/manager/IManager.java | 3 +
.../iotdb/confignode/manager/ModelManager.java | 5 +
.../manager/load/balancer/RouteBalancer.java | 152 ++++++---
.../iotdb/confignode/manager/node/NodeManager.java | 8 +
.../iotdb/confignode/persistence/ModelInfo.java | 2 +-
.../procedure/impl/StateMachineProcedure.java | 9 +-
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 2 +
.../iotdb/confignode/service/ConfigNode.java | 44 +--
.../consensus/config/PipeConsensusConfig.java | 21 +-
.../consensus/pipe/PipeConsensusServerImpl.java | 21 +-
...xManager.java => ReplicateProgressManager.java} | 8 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 71 ++++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 286 +----------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 105 ++++---
.../db/consensus/DataRegionConsensusImpl.java | 10 +-
.../client/IoTDBDataNodeAsyncClientManager.java | 4 +
.../evolvable/batch/PipeTabletEventBatch.java | 37 ++-
.../evolvable/batch/PipeTabletEventPlainBatch.java | 41 +--
.../batch/PipeTabletEventTsFileBatch.java | 11 +-
.../pipeconsensus/PipeConsensusAsyncConnector.java | 16 +-
.../pipeconsensus/PipeConsensusSyncConnector.java | 9 +-
.../PipeConsensusTabletInsertionEventHandler.java | 8 +-
.../PipeConsensusTsFileInsertionEventHandler.java | 8 +-
.../PipeConsensusTransferBatchReqBuilder.java | 5 +-
.../async/IoTDBDataRegionAsyncConnector.java | 1 +
....java => ReplicateProgressDataNodeManager.java} | 36 ++-
.../deletion/DeletionResourceManager.java | 4 +-
.../deletion/persist/PageCacheDeletionBuffer.java | 5 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 17 +-
.../event/realtime/PipeRealtimeEventFactory.java | 71 ++++-
...oricalDataRegionTsFileAndDeletionExtractor.java | 17 ++
.../realtime/assigner/PipeDataRegionAssigner.java | 2 +
.../listener/PipeInsertionDataNodeListener.java | 16 +-
.../pipeconsensus/PipeConsensusProcessor.java | 43 ++-
.../pipeconsensus/PipeConsensusReceiver.java | 338 +++++++++------------
.../db/pipe/resource/memory/PipeMemoryManager.java | 5 +-
.../execution/memory/LocalMemoryManager.java | 3 +-
.../analyze/cache/partition/PartitionCache.java | 4 +-
.../cache/schema/DataNodeDevicePathCache.java | 4 +-
.../plan/planner/LocalExecutionPlanner.java | 6 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 4 +-
.../metric/SchemaEngineCachedMetric.java | 7 +-
.../rescon/MemSchemaEngineStatistics.java | 5 +-
.../ReleaseFlushStrategySizeBasedImpl.java | 7 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 64 +---
.../iotdb/db/service/ExternalRPCService.java | 2 +-
...viceMBean.java => ExternalRPCServiceMBean.java} | 2 +-
.../metrics/memory/ConsensusMemoryMetrics.java | 6 +-
.../metrics/memory/GlobalMemoryMetrics.java | 6 +-
.../metrics/memory/OffHeapMemoryMetrics.java | 6 +-
.../metrics/memory/QueryEngineMemoryMetrics.java | 30 +-
.../metrics/memory/SchemaEngineMemoryMetrics.java | 16 +-
.../metrics/memory/StorageEngineMemoryMetrics.java | 28 +-
.../metrics/memory/StreamEngineMemoryMetrics.java | 6 +-
.../iotdb/db/storageengine/StorageEngine.java | 19 +-
.../db/storageengine/buffer/BloomFilterCache.java | 4 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 6 +-
.../buffer/TimeSeriesMetadataCache.java | 4 +-
.../storageengine/dataregion/DataRegionInfo.java | 6 +-
.../dataregion/memtable/AbstractMemTable.java | 61 +++-
.../memtable/AlignedWritableMemChunk.java | 13 +
.../memtable/AlignedWritableMemChunkGroup.java | 15 +-
.../dataregion/memtable/IMemTable.java | 2 +
.../memtable/IWritableMemChunkGroup.java | 2 +
.../dataregion/memtable/TsFileProcessor.java | 41 ++-
.../dataregion/memtable/WritableMemChunk.java | 8 +
.../dataregion/memtable/WritableMemChunkGroup.java | 20 ++
.../dataregion/tsfile/TsFileResource.java | 8 +-
.../dataregion/wal/utils/WALInsertNodeCache.java | 4 +-
.../rescon/memory/PrimitiveArrayManager.java | 5 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 18 +-
.../rescon/memory/TimePartitionManager.java | 5 +-
.../rescon/memory/TsFileResourceManager.java | 6 +-
.../db/utils/datastructure/AlignedTVList.java | 44 ++-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 38 ++-
.../db/utils/datastructure/BooleanTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/FloatTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/IntTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/LongTVList.java | 38 ++-
.../iotdb/db/utils/datastructure/TVList.java | 70 ++++-
.../fetcher/cache/TableDeviceSchemaCacheTest.java | 10 +-
.../dataregion/memtable/TsFileProcessorTest.java | 36 +--
.../rescon/memory/ResourceManagerTest.java | 5 +-
.../rescon/memory/TimePartitionManagerTest.java | 7 +-
.../datastructure/PrimitiveArrayManagerTest.java | 5 +-
.../async/AsyncPipeDataTransferServiceClient.java | 14 +
.../commons/memory/AtomicLongMemoryBlock.java | 36 +--
.../apache/iotdb/commons/memory/IMemoryBlock.java | 18 +-
.../apache/iotdb/commons/memory/MemoryConfig.java | 315 +++++++++++++++++++
.../iotdb/commons/memory/MemoryException.java | 7 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 33 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 17 +-
.../iotdb/commons/memory/MemoryManagerTest.java | 2 +-
.../thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/pipeconsensus.thrift | 5 +-
103 files changed, 1729 insertions(+), 1166 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index fffc290ba7a,9e0dbf57f3a..4fec25153ba
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@@ -2221,22 -2218,22 +2218,22 @@@ public class IoTDBDescriptor
// if pipe proportion is set, use it, otherwise use the default value
if (proportions.length >= 6) {
pipeMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum;
} else {
pipeMemorySize =
- (maxMemoryAvailable
- - (memoryConfig.getStorageEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getQueryEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes()
- + memoryConfig.getConsensusMemoryManager().getTotalMemorySizeInBytes()))
+ (memoryAvailable
- - (conf.getStorageEngineMemoryManager().getTotalMemorySizeInBytes()
- + conf.getQueryEngineMemoryManager().getTotalMemorySizeInBytes()
- + conf.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes()
- + conf.getConsensusMemoryManager().getTotalMemorySizeInBytes()))
++ - (memoryConfig.getStorageEngineMemoryManager().getTotalMemorySizeInBytes()
++ + memoryConfig.getQueryEngineMemoryManager().getTotalMemorySizeInBytes()
++ + memoryConfig.getSchemaEngineMemoryManager().getTotalMemorySizeInBytes()
++ + memoryConfig.getConsensusMemoryManager().getTotalMemorySizeInBytes()))
/ 2;
}
}
}
// on heap memory manager
MemoryManager onheapMemoryManager =
- globalMemoryManager.getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().totalMemory());
- conf.setOnHeapMemoryManager(onheapMemoryManager);
- MemoryConfig.global().getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().maxMemory());
++ MemoryConfig.global().getOrCreateMemoryManager("OnHeap", Runtime.getRuntime().totalMemory());
+ memoryConfig.setOnHeapMemoryManager(onheapMemoryManager);
// storage engine memory manager
MemoryManager storageEngineMemoryManager =
onheapMemoryManager.getOrCreateMemoryManager("StorageEngine", storageEngineMemorySize);
@@@ -2467,20 -2464,20 +2464,20 @@@
if (proportionSum != 0) {
try {
bloomFilterCacheMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[0].trim()) / proportionSum;
chunkCacheMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[1].trim()) / proportionSum;
timeSeriesMetaDataCacheMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[2].trim()) / proportionSum;
coordinatorMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[3].trim()) / proportionSum;
operatorsMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[4].trim()) / proportionSum;
dataExchangeMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[5].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[5].trim()) / proportionSum;
timeIndexMemorySize =
- maxMemoryAvailable * Integer.parseInt(proportions[6].trim()) / proportionSum;
+ memoryAvailable * Integer.parseInt(proportions[6].trim()) / proportionSum;
- } catch (Exception e) {
+ } catch (IllegalArgumentException e) {
throw new RuntimeException(
"Each subsection of configuration item chunkmeta_chunk_timeseriesmeta_free_memory_proportion"
+ " should be an integer, which is "
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
index 5c7f5ad7242,d60920c0b31..b71b285e88a
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/consensus/DataRegionConsensusImpl.java
@@@ -113,7 -115,9 +115,9 @@@ public class DataRegionConsensusImpl
private static ConsensusConfig buildConsensusConfig() {
IMemoryBlock memoryBlock =
- CONF.getConsensusMemoryManager().forceAllocate("Consensus", MemoryBlockType.DYNAMIC);
+ MEMORY_CONFIG
+ .getConsensusMemoryManager()
- .forceAllocate("Consensus", MemoryBlockType.FUNCTION);
++ .forceAllocate("Consensus", MemoryBlockType.DYNAMIC);
return ConsensusConfig.newBuilder()
.setThisNodeId(CONF.getDataNodeId())
.setThisNode(new TEndPoint(CONF.getInternalAddress(), CONF.getDataRegionConsensusPort()))
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
index bede51da24d,e72c8ff3375..80784009b08
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/resource/memory/PipeMemoryManager.java
@@@ -53,10 -53,9 +53,9 @@@ public class PipeMemoryManager
// TODO @spricoder: consider combine memory block and used MemorySizeInBytes
private IMemoryBlock memoryBlock =
- IoTDBDescriptor.getInstance()
- .getConfig()
+ MemoryConfig.getInstance()
.getPipeMemoryManager()
- .forceAllocate("Stream", MemoryBlockType.FUNCTION);
+ .forceAllocate("Stream", MemoryBlockType.DYNAMIC);
private static final double EXCEED_PROTECT_THRESHOLD = 0.95;
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
index 143d92e62fa,17f0a82dda5..e1de5c71acb
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/partition/PartitionCache.java
@@@ -119,9 -121,9 +121,9 @@@ public class PartitionCache
public PartitionCache() {
this.memoryBlock =
- config
+ memoryConfig
.getPartitionCacheMemoryManager()
- .forceAllocate("PartitionCache", MemoryBlockType.FUNCTION);
+ .forceAllocate("PartitionCache", MemoryBlockType.STATIC);
this.memoryBlock.allocate(this.memoryBlock.getTotalMemorySizeInBytes());
// TODO @spricoder: PartitionCache need to be controlled according to memory
this.schemaPartitionCache =
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
index fca17a11430,bd3c49bf998..ce3d1bc1737
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/cache/schema/DataNodeDevicePathCache.java
@@@ -43,9 -45,9 +45,9 @@@ public class DataNodeDevicePathCache
private DataNodeDevicePathCache() {
devicePathCacheMemoryBlock =
- config
+ memoryConfig
.getDevicePathCacheMemoryManager()
- .forceAllocate("DevicePathCache", MemoryBlockType.PERFORMANCE);
+ .forceAllocate("DevicePathCache", MemoryBlockType.STATIC);
// TODO @spricoder: later we can find a way to get the byte size of cache
devicePathCacheMemoryBlock.allocate(devicePathCacheMemoryBlock.getTotalMemorySizeInBytes());
devicePathCache =
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
index 93f45bebb5e,b380f34ce89..1f799e7401b
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/LocalExecutionPlanner.java
@@@ -67,9 -68,12 +68,12 @@@ public class LocalExecutionPlanner
static {
IoTDBConfig CONFIG = IoTDBDescriptor.getInstance().getConfig();
+ MemoryConfig MEMORY_CONFIG = MemoryConfig.getInstance();
OPERATORS_MEMORY_BLOCK =
- CONFIG.getOperatorsMemoryManager().forceAllocate("Operators", MemoryBlockType.DYNAMIC);
+ MEMORY_CONFIG
+ .getOperatorsMemoryManager()
- .forceAllocate("Operators", MemoryBlockType.FUNCTION);
++ .forceAllocate("Operators", MemoryBlockType.DYNAMIC);
MIN_REST_MEMORY_FOR_QUERY_AFTER_LOAD =
(long)
((OPERATORS_MEMORY_BLOCK.getTotalMemorySizeInBytes())
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
index 604208b1edc,06233626a28..1380c06e883
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/fetcher/cache/TableDeviceSchemaCache.java
@@@ -106,9 -108,9 +108,9 @@@ public class TableDeviceSchemaCache
private TableDeviceSchemaCache() {
memoryBlock =
- config
+ memoryConfig
.getSchemaCacheMemoryManager()
- .forceAllocate("TableDeviceSchemaCache", MemoryBlockType.FUNCTION);
+ .forceAllocate("TableDeviceSchemaCache", MemoryBlockType.STATIC);
dualKeyCache =
new DualKeyCacheBuilder<TableId, IDeviceID, TableDeviceCacheEntry>()
.cacheEvictionPolicy(
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java
index 51d6ad337a7,32d3c9c2f75..e6c44c52e43
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/rescon/MemSchemaEngineStatistics.java
@@@ -56,10 -56,9 +56,9 @@@ public class MemSchemaEngineStatistics
public MemSchemaEngineStatistics() {
memoryBlock =
- IoTDBDescriptor.getInstance()
- .getConfig()
+ MemoryConfig.getInstance()
.getSchemaRegionMemoryManager()
- .forceAllocate("SchemaRegion", MemoryBlockType.FUNCTION);
+ .forceAllocate("SchemaRegion", MemoryBlockType.DYNAMIC);
}
@Override
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
index be2a48c19aa,be23fe45db5..8dbf8691310
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/BloomFilterCache.java
@@@ -58,9 -60,9 +60,9 @@@ public class BloomFilterCache
static {
CACHE_MEMORY_BLOCK =
- CONFIG
+ MEMORY_CONFIG
.getBloomFilterCacheMemoryManager()
- .forceAllocate("BloomFilterCache", MemoryBlockType.PERFORMANCE);
+ .forceAllocate("BloomFilterCache", MemoryBlockType.STATIC);
// TODO @spricoder: find a way to get the size of the BloomFilterCache
CACHE_MEMORY_BLOCK.allocate(CACHE_MEMORY_BLOCK.getTotalMemorySizeInBytes());
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
index d964a635b8e,301480239bd..30d17a41ce0
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/ChunkCache.java
@@@ -73,7 -75,9 +75,9 @@@ public class ChunkCache
static {
CACHE_MEMORY_BLOCK =
- CONFIG.getChunkCacheMemoryManager().forceAllocate("ChunkCache", MemoryBlockType.STATIC);
+ MEMORY_CONFIG
+ .getChunkCacheMemoryManager()
- .forceAllocate("ChunkCache", MemoryBlockType.PERFORMANCE);
++ .forceAllocate("ChunkCache", MemoryBlockType.STATIC);
// TODO @spricoder: find a way to get the size of the ChunkCache
CACHE_MEMORY_BLOCK.allocate(CACHE_MEMORY_BLOCK.getTotalMemorySizeInBytes());
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
index 3ba8e8a4af3,b7d85968f31..3e09d144a1f
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java
@@@ -85,9 -87,9 +87,9 @@@ public class TimeSeriesMetadataCache
static {
CACHE_MEMORY_BLOCK =
- config
+ memoryConfig
.getTimeSeriesMetaDataCacheMemoryManager()
- .forceAllocate("TimeSeriesMetadataCache", MemoryBlockType.PERFORMANCE);
+ .forceAllocate("TimeSeriesMetadataCache", MemoryBlockType.STATIC);
// TODO @spricoder find a better way to get the size of cache
CACHE_MEMORY_BLOCK.allocate(CACHE_MEMORY_BLOCK.getTotalMemorySizeInBytes());
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
index 1b9b523dd12,ad73a1bdc9c..95095655312
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/PrimitiveArrayManager.java
@@@ -53,9 -56,9 +56,9 @@@ public class PrimitiveArrayManager
/** memory block for all arrays */
private static final IMemoryBlock POOLED_ARRAYS_MEMORY_BLOCK =
- CONFIG
+ MEMORY_CONFIG
.getBufferedArraysMemoryManager()
- .forceAllocate("BufferedArrays", MemoryBlockType.FUNCTION);
+ .forceAllocate("BufferedArrays", MemoryBlockType.DYNAMIC);
/** threshold total size of arrays for all data types */
private static final double POOLED_ARRAYS_MEMORY_THRESHOLD =
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java
index 7819e9b5a53,ba8d704d9fb..516ac0541cd
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/SystemInfo.java
@@@ -46,10 -47,11 +47,13 @@@ import java.util.concurrent.atomic.Atom
import java.util.concurrent.atomic.AtomicLong;
public class SystemInfo {
+
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
private static final Logger logger = LoggerFactory.getLogger(SystemInfo.class);
+ private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+ private static final MemoryConfig memoryConfig = MemoryConfig.getInstance();
+
private long totalStorageGroupMemCost = 0L;
private volatile boolean rejected = false;
@@@ -78,13 -80,17 +82,17 @@@
private SystemInfo() {
compactionMemoryBlock =
- config.getCompactionMemoryManager().forceAllocate("Compaction", MemoryBlockType.DYNAMIC);
+ memoryConfig
+ .getCompactionMemoryManager()
- .forceAllocate("Compaction", MemoryBlockType.FUNCTION);
++ .forceAllocate("Compaction", MemoryBlockType.DYNAMIC);
walBufferQueueMemoryBlock =
- config.getWalBufferQueueManager().forceAllocate("WalBufferQueue", MemoryBlockType.DYNAMIC);
+ memoryConfig
+ .getWalBufferQueueManager()
- .forceAllocate("WalBufferQueue", MemoryBlockType.FUNCTION);
++ .forceAllocate("WalBufferQueue", MemoryBlockType.DYNAMIC);
directBufferMemoryBlock =
- config
+ memoryConfig
.getDirectBufferMemoryManager()
- .forceAllocate("DirectBuffer", MemoryBlockType.FUNCTION);
+ .forceAllocate("DirectBuffer", MemoryBlockType.DYNAMIC);
loadWriteMemory();
}
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
index 85e96683413,2d8b2206c06..53f44867ff2
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TimePartitionManager.java
@@@ -48,10 -48,9 +48,9 @@@ public class TimePartitionManager
private TimePartitionManager() {
timePartitionInfoMap = new HashMap<>();
timePartitionInfoMemoryBlock =
- IoTDBDescriptor.getInstance()
- .getConfig()
+ MemoryConfig.getInstance()
.getTimePartitionInfoMemoryManager()
- .forceAllocate("TimePartitionInfoMemoryBlock", MemoryBlockType.FUNCTION);
+ .forceAllocate("TimePartitionInfoMemoryBlock", MemoryBlockType.DYNAMIC);
}
public void registerTimePartitionInfo(TimePartitionInfo timePartitionInfo) {
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
index 2cecc1ed3a6,50fbc3a62c2..9d929967ab7
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java
@@@ -49,7 -51,9 +51,9 @@@ public class TsFileResourceManager
private TsFileResourceManager() {
memoryBlock =
- CONFIG.getTimeIndexMemoryManager().forceAllocate("TimeIndex", MemoryBlockType.DYNAMIC);
+ MEMORY_CONFIG
+ .getTimeIndexMemoryManager()
- .forceAllocate("TimeIndex", MemoryBlockType.FUNCTION);
++ .forceAllocate("TimeIndex", MemoryBlockType.DYNAMIC);
}
@TestOnly
diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
index d577d2e90bc,360418303fe..1fbf9f5f237
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
@@@ -73,11 -69,9 +73,10 @@@ public class MemoryManager
this.enable = false;
}
- private MemoryManager(
- String name, MemoryManager parentMemoryManager, long totalMemorySizeInBytes) {
+ MemoryManager(String name, MemoryManager parentMemoryManager, long totalMemorySizeInBytes) {
this.name = name;
this.parentMemoryManager = parentMemoryManager;
+ this.totalAllocatedMemorySizeInBytes = totalMemorySizeInBytes;
this.totalMemorySizeInBytes = totalMemorySizeInBytes;
this.enable = false;
}