You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sp...@apache.org on 2025/03/07 04:25:15 UTC
(iotdb) branch feature/memory_transfer updated (fbe68fd8755 -> 62f94d73741)
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a change to branch feature/memory_transfer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from fbe68fd8755 Merge branch 'feature/memory_auto' into feature/memory_transfer
add 0db89873e78 Modified pks, sversion, dbname (#14999)
add 1adc74dffd2 fix: memtable enhancement issues (#14994)
add e8d07c156da Load: detect region migration during second phase (#15005)
add 17f099e03db Fix typo in SimpleNestedLoopJoinOperator
add 5911f231b79 Pipe: Using caching to optimize the verifyTableDataType method in the Load analysis phase (#15007)
add 9154e7c67c8 revert: page access when initialize statistics and build TsBlock for aligned memchunk (#15013)
add 83d53722cd3 Make the error message clearer for query operator
add d412a202af6 Reject insert without field column (#15017)
add f9f0574b0f7 Fix frozen IoTConsensus sync lag (#15023)
add 8fe854752de Bump jinja2 from 3.1.5 to 3.1.6 in /iotdb-core/ainode (#15026)
add 1ab9631cb4b Added comment for table & column
add 93f0795233d IoTConsensusV2: persistent isGeneratedByPipeConsensus (#14938)
add 1669f503907 [Fix](query)(table model) GroupedAccumulator are not discovered before because of stream optimize
add b8947ff54fd Delete Maintain Auth
add 809ecd4cad1 Fix (#15027)
add 23068b42bbe Add memory metrics and refact memory allocation (#14710)
add f58ba839f2e Fixed the memory leakage of TableDeviceSchemaCache & The invalidation logic when filter by SK & The bug that last time may not be invalidated when the measurement with largest last time is invalidated (#15018)
add fddf0a6eaa2 Fixed the metadata query & Fragment Instance & retry's timeout
new 62f94d73741 Merge
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 -----
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 17 +-
.../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 -
.../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 11 +-
.../manual/basic/IoTDBPipeLifeCycleIT.java | 54 +---
.../manual/enhanced/IoTDBPipeIdempotentIT.java | 12 +
.../it/db/it/IoTDBAuthenticationTableIT.java | 114 +++++++-
.../relational/it/db/it/IoTDBInsertTableIT.java | 40 +--
.../scalar/IoTDBFormatFunctionTableIT.java | 2 +-
.../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +-
.../it/query/recent/IoTDBMaintainAuthIT.java | 89 +++---
.../it/query/recent/IoTDBTableAggregationIT.java | 4 +-
.../IoTDBTableAggregationNonStreamIT.java} | 27 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 53 ++--
.../iotdb/relational/it/schema/IoTDBTableIT.java | 37 ++-
.../it/session/IoTDBSessionRelationalIT.java | 141 ++++++---
.../it/local/IoTDBSubscriptionBasicIT.java | 38 +--
.../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 88 ++----
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 7 +
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 64 +++++
.../IoTDBRelationalDatabaseMetadata.java | 94 +++++-
.../base/AbstractSubscriptionConsumer.java | 7 +-
iotdb-core/ainode/poetry.lock | 149 +++++++---
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../request/ConfigPhysicalPlanVisitor.java | 15 +
...umnPlan.java => SetTableColumnCommentPlan.java} | 26 +-
...bleColumnPlan.java => SetTableCommentPlan.java} | 27 +-
.../iotdb/confignode/manager/ConfigManager.java | 14 +
.../manager/load/balancer/PartitionBalancer.java | 8 +-
.../iotdb/confignode/manager/node/NodeManager.java | 45 +--
.../extractor/ConfigRegionListeningFilter.java | 6 +-
...ConfigPhysicalPlanTablePatternParseVisitor.java | 22 ++
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 118 ++++++--
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 36 ++-
.../manager/schema/ClusterSchemaManager.java | 40 ++-
.../persistence/executor/ConfigPlanExecutor.java | 6 +
.../persistence/schema/ClusterSchemaInfo.java | 39 +++
.../confignode/persistence/schema/ConfigMTree.java | 31 ++
.../request/ConfigPhysicalPlanSerDeTest.java | 33 ++-
...igPhysicalPlanTablePatternParseVisitorTest.java | 18 ++
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +++---
.../consensus/pipe/PipeConsensusServerImpl.java | 4 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 6 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 6 +-
.../event/realtime/PipeRealtimeEventFactory.java | 5 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 1 -
.../listener/PipeInsertionDataNodeListener.java | 8 +-
.../pipeconsensus/PipeConsensusReceiver.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 8 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 36 ++-
.../queryengine/execution/memory/MemoryPool.java | 8 +-
.../execution/operator/AbstractOperator.java | 5 +-
.../join/SimpleNestedLoopCrossJoinOperator.java | 2 +-
.../InformationSchemaContentSupplierFactory.java | 13 +
.../aggregation/grouped/GroupedAggregator.java | 4 +-
.../grouped/GroupedFirstAccumulator.java | 11 +-
.../grouped/GroupedFirstByAccumulator.java | 2 +-
.../grouped/GroupedLastAccumulator.java | 1 +
.../grouped/GroupedLastByAccumulator.java | 2 +-
.../grouped/GroupedVarianceAccumulator.java | 2 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 8 +-
.../plan/analyze/ClusterPartitionFetcher.java | 5 +
.../plan/analyze/IPartitionFetcher.java | 4 +
.../analyze/cache/partition/PartitionCache.java | 6 +-
.../cache/schema/DataNodeDevicePathCache.java | 2 +-
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +
.../analyze/load/LoadTsFileTableSchemaCache.java | 24 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +-
.../queryengine/plan/execution/QueryExecution.java | 4 -
.../execution/config/TableConfigTaskVisitor.java | 140 ++++++---
.../config/executor/ClusterConfigTaskExecutor.java | 85 ++++++
.../config/executor/IConfigTaskExecutor.java | 16 ++
...nTask.java => AlterTableCommentColumnTask.java} | 13 +-
...esTask.java => AlterTableCommentTableTask.java} | 19 +-
.../relational/DescribeTableDetailsTask.java | 6 +
.../metadata/relational/ShowTablesDetailsTask.java | 7 +
.../plan/planner/LocalExecutionPlanner.java | 8 +-
.../SimpleFragmentParallelPlanner.java | 2 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../plan/planner/plan/node/write/InsertNode.java | 3 +-
.../plan/node/write/RelationalInsertRowNode.java | 5 -
.../analyzer/StatementAnalyzerFactory.java | 4 +
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 6 +-
.../fetcher/TableDeviceSchemaValidator.java | 6 +-
.../fetcher/TableHeaderSchemaValidator.java | 47 ++-
.../fetcher/cache/TableAttributeSchema.java | 6 +-
.../fetcher/cache/TableDeviceLastCache.java | 5 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 6 +-
.../distribute/TableModelQueryFragmentPlanner.java | 2 +-
.../plan/relational/security/AccessControl.java | 8 -
.../relational/security/AccessControlImpl.java | 6 +-
.../relational/security/AllowAllAccessControl.java | 5 -
.../relational/security/TableModelPrivilege.java | 5 -
.../plan/relational/sql/ast/AstVisitor.java | 12 +
.../plan/relational/sql/ast/ColumnDefinition.java | 27 +-
.../plan/relational/sql/ast/CreateTable.java | 8 +
.../ast/{DropColumn.java => SetColumnComment.java} | 26 +-
.../plan/relational/sql/ast/SetProperties.java | 12 -
.../{CountStatement.java => SetTableComment.java} | 60 ++--
...onfiguration.java => ShowQueriesStatement.java} | 18 +-
.../plan/relational/sql/parser/AstBuilder.java | 29 +-
.../plan/relational/sql/rewrite/ShowRewrite.java | 26 +-
.../sql/rewrite/StatementRewriteFactory.java | 6 +-
.../plan/relational/sql/util/SqlFormatter.java | 42 ++-
.../plan/scheduler/load/LoadTsFileScheduler.java | 117 +++++---
.../rescon/MemSchemaEngineStatistics.java | 4 +-
.../db/storageengine/buffer/BloomFilterCache.java | 2 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 2 +-
.../buffer/TimeSeriesMetadataCache.java | 2 +-
.../execute/task/CrossSpaceCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../task/RepairUnsortedFileCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 12 +-
.../dataregion/memtable/AbstractMemTable.java | 9 -
.../memtable/AlignedReadOnlyMemChunk.java | 318 +++++++--------------
.../memtable/AlignedWritableMemChunk.java | 134 +++------
.../memtable/AlignedWritableMemChunkGroup.java | 5 -
.../dataregion/memtable/IMemTable.java | 2 -
.../memtable/IWritableMemChunkGroup.java | 2 -
.../dataregion/memtable/ReadOnlyMemChunk.java | 16 +-
.../dataregion/memtable/TsFileProcessor.java | 12 +-
.../dataregion/memtable/WritableMemChunk.java | 4 +-
.../dataregion/memtable/WritableMemChunkGroup.java | 8 -
.../read/reader/chunk/MemAlignedChunkReader.java | 188 ++++--------
.../read/reader/chunk/MemChunkReader.java | 4 +-
.../read/reader/chunk/MemPageReader.java | 6 +-
.../dataregion/tsfile/TsFileResource.java | 19 ++
.../dataregion/wal/buffer/IWALBuffer.java | 10 +
.../dataregion/wal/buffer/WALBuffer.java | 13 +
.../storageengine/dataregion/wal/node/WALNode.java | 8 +-
.../memory/LoadTsFileDataCacheMemoryBlock.java | 2 +-
.../load/memory/LoadTsFileMemoryManager.java | 2 +-
.../rescon/memory/PrimitiveArrayManager.java | 2 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 6 +-
.../rescon/memory/TimePartitionManager.java | 5 +-
.../rescon/memory/TsFileResourceManager.java | 4 +-
.../db/utils/datastructure/AlignedTVList.java | 55 ++--
.../datastructure/FixedPriorityBlockingQueue.java | 2 +-
.../MergeSortAlignedTVListIterator.java | 147 +++-------
...tIterator.java => MergeSortTVListIterator.java} | 63 ++--
.../utils/datastructure/PageColumnAccessInfo.java | 52 ----
.../iotdb/db/utils/datastructure/TVList.java | 4 +-
.../pipe/event/TsFileInsertionEventParserTest.java | 1 -
.../plan/analyze/FakePartitionFetcherImpl.java | 5 +
.../plan/planner/distribution/Util.java | 5 +
.../plan/planner/distribution/Util2.java | 5 +
.../plan/relational/analyzer/AnalyzerTest.java | 4 +-
.../plan/relational/analyzer/TSBSMetadata.java | 7 +
.../plan/relational/analyzer/TestMatadata.java | 7 +
.../plan/relational/planner/PlanTester.java | 9 +-
.../relational/sql/parser/AuthorStatementTest.java | 5 -
.../CompactionTableModelTestFileWriter.java | 14 +
.../read/reader/chunk/MemChunkLoaderTest.java | 26 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +-
.../commons/memory/AtomicLongMemoryBlock.java | 2 +-
.../apache/iotdb/commons/memory/IMemoryBlock.java | 13 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 146 +++++-----
.../schema/column/ColumnHeaderConstant.java | 7 +-
.../table/AlterOrDropTableOperationType.java | 8 +-
.../commons/schema/table/InformationSchema.java | 6 +
.../apache/iotdb/commons/schema/table/TsTable.java | 1 +
.../iotdb/commons/memory/MemoryManagerTest.java | 14 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 19 +-
.../src/main/thrift/confignode.thrift | 1 +
169 files changed, 2392 insertions(+), 1748 deletions(-)
copy integration-test/src/test/java/org/apache/iotdb/relational/it/query/{old/aligned/IoTDBAlignedSeriesQueryTable5IT.java => recent/IoTDBTableAggregationNonStreamIT.java} (61%)
copy iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{AbstractTableColumnPlan.java => SetTableColumnCommentPlan.java} (71%)
copy iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/table/{AbstractTableColumnPlan.java => SetTableCommentPlan.java} (70%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{AlterTableDropColumnTask.java => AlterTableCommentColumnTask.java} (83%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/relational/{AlterTableSetPropertiesTask.java => AlterTableCommentTableTask.java} (77%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{DropColumn.java => SetColumnComment.java} (82%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{CountStatement.java => SetTableComment.java} (60%)
copy iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/{LoadConfiguration.java => ShowQueriesStatement.java} (69%)
rename iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/{MergeSortTvListIterator.java => MergeSortTVListIterator.java} (67%)
delete mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/datastructure/PageColumnAccessInfo.java
(iotdb) 01/01: Merge
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spricoder pushed a commit to branch feature/memory_transfer
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 62f94d7374159dbe2cd50b2c9fa46c846a5d95ed
Merge: fbe68fd8755 fddf0a6eaa2
Author: spricoder <sp...@qq.com>
AuthorDate: Fri Mar 7 12:25:04 2025 +0800
Merge
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 73 -----
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 17 +-
.../it/auth/IoTDBClusterAuthorityRelationalIT.java | 1 -
.../iotdb/db/it/auth/IoTDBRelationalAuthIT.java | 11 +-
.../manual/basic/IoTDBPipeLifeCycleIT.java | 54 +---
.../manual/enhanced/IoTDBPipeIdempotentIT.java | 12 +
.../it/db/it/IoTDBAuthenticationTableIT.java | 114 +++++++-
.../relational/it/db/it/IoTDBInsertTableIT.java | 40 +--
.../scalar/IoTDBFormatFunctionTableIT.java | 2 +-
.../query/old/query/IoTDBGreatestLeastTableIT.java | 2 +-
.../it/query/recent/IoTDBMaintainAuthIT.java | 89 +++---
.../it/query/recent/IoTDBTableAggregationIT.java | 4 +-
.../recent/IoTDBTableAggregationNonStreamIT.java | 56 ++++
.../relational/it/schema/IoTDBDatabaseIT.java | 53 ++--
.../iotdb/relational/it/schema/IoTDBTableIT.java | 37 ++-
.../it/session/IoTDBSessionRelationalIT.java | 141 ++++++---
.../it/local/IoTDBSubscriptionBasicIT.java | 38 +--
.../iotdb/jdbc/IoTDBAbstractDatabaseMetadata.java | 88 ++----
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 7 +
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 64 +++++
.../IoTDBRelationalDatabaseMetadata.java | 94 +++++-
.../base/AbstractSubscriptionConsumer.java | 7 +-
iotdb-core/ainode/poetry.lock | 149 +++++++---
.../consensus/request/ConfigPhysicalPlan.java | 8 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../request/ConfigPhysicalPlanVisitor.java | 15 +
.../write/table/SetTableColumnCommentPlan.java | 62 ++++
.../request/write/table/SetTableCommentPlan.java | 57 ++++
.../iotdb/confignode/manager/ConfigManager.java | 14 +
.../manager/load/balancer/PartitionBalancer.java | 8 +-
.../iotdb/confignode/manager/node/NodeManager.java | 45 +--
.../extractor/ConfigRegionListeningFilter.java | 6 +-
...ConfigPhysicalPlanTablePatternParseVisitor.java | 22 ++
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 118 ++++++--
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 36 ++-
.../manager/schema/ClusterSchemaManager.java | 40 ++-
.../persistence/executor/ConfigPlanExecutor.java | 6 +
.../persistence/schema/ClusterSchemaInfo.java | 39 +++
.../confignode/persistence/schema/ConfigMTree.java | 31 ++
.../request/ConfigPhysicalPlanSerDeTest.java | 33 ++-
...igPhysicalPlanTablePatternParseVisitorTest.java | 18 ++
.../apache/iotdb/consensus/pipe/PipeConsensus.java | 80 +++---
.../consensus/pipe/PipeConsensusServerImpl.java | 4 +-
.../pipe/metric/PipeConsensusSyncLagManager.java | 6 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../db/consensus/DataRegionConsensusImpl.java | 2 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 6 +-
.../event/realtime/PipeRealtimeEventFactory.java | 5 +-
...oricalDataRegionTsFileAndDeletionExtractor.java | 1 -
.../listener/PipeInsertionDataNodeListener.java | 8 +-
.../pipeconsensus/PipeConsensusReceiver.java | 2 +-
.../db/pipe/resource/memory/PipeMemoryManager.java | 8 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 36 ++-
.../queryengine/execution/memory/MemoryPool.java | 8 +-
.../execution/operator/AbstractOperator.java | 5 +-
.../join/SimpleNestedLoopCrossJoinOperator.java | 2 +-
.../InformationSchemaContentSupplierFactory.java | 13 +
.../aggregation/grouped/GroupedAggregator.java | 4 +-
.../grouped/GroupedFirstAccumulator.java | 11 +-
.../grouped/GroupedFirstByAccumulator.java | 2 +-
.../grouped/GroupedLastAccumulator.java | 1 +
.../grouped/GroupedLastByAccumulator.java | 2 +-
.../grouped/GroupedVarianceAccumulator.java | 2 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 8 +-
.../plan/analyze/ClusterPartitionFetcher.java | 5 +
.../plan/analyze/IPartitionFetcher.java | 4 +
.../analyze/cache/partition/PartitionCache.java | 6 +-
.../cache/schema/DataNodeDevicePathCache.java | 2 +-
.../schema/dualkeycache/impl/DualKeyCacheImpl.java | 9 +-
.../plan/analyze/load/LoadTsFileAnalyzer.java | 4 +
.../analyze/load/LoadTsFileTableSchemaCache.java | 24 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 6 +-
.../analyze/schema/ClusterSchemaFetchExecutor.java | 7 +-
.../queryengine/plan/execution/QueryExecution.java | 4 -
.../execution/config/TableConfigTaskVisitor.java | 140 ++++++---
.../config/executor/ClusterConfigTaskExecutor.java | 85 ++++++
.../config/executor/IConfigTaskExecutor.java | 16 ++
.../relational/AlterTableCommentColumnTask.java | 53 ++++
.../relational/AlterTableCommentTableTask.java | 48 ++++
.../relational/DescribeTableDetailsTask.java | 6 +
.../metadata/relational/ShowTablesDetailsTask.java | 7 +
.../plan/planner/LocalExecutionPlanner.java | 8 +-
.../SimpleFragmentParallelPlanner.java | 2 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../plan/planner/plan/node/write/InsertNode.java | 3 +-
.../plan/node/write/RelationalInsertRowNode.java | 5 -
.../analyzer/StatementAnalyzerFactory.java | 4 +
.../metadata/fetcher/TableDeviceSchemaFetcher.java | 6 +-
.../fetcher/TableDeviceSchemaValidator.java | 6 +-
.../fetcher/TableHeaderSchemaValidator.java | 47 ++-
.../fetcher/cache/TableAttributeSchema.java | 6 +-
.../fetcher/cache/TableDeviceLastCache.java | 5 +-
.../fetcher/cache/TableDeviceSchemaCache.java | 6 +-
.../distribute/TableModelQueryFragmentPlanner.java | 2 +-
.../plan/relational/security/AccessControl.java | 8 -
.../relational/security/AccessControlImpl.java | 6 +-
.../relational/security/AllowAllAccessControl.java | 5 -
.../relational/security/TableModelPrivilege.java | 5 -
.../plan/relational/sql/ast/AstVisitor.java | 12 +
.../plan/relational/sql/ast/ColumnDefinition.java | 27 +-
.../plan/relational/sql/ast/CreateTable.java | 8 +
.../plan/relational/sql/ast/SetColumnComment.java | 116 ++++++++
.../plan/relational/sql/ast/SetProperties.java | 12 -
.../plan/relational/sql/ast/SetTableComment.java | 98 +++++++
.../relational/sql/ast/ShowQueriesStatement.java} | 40 +--
.../plan/relational/sql/parser/AstBuilder.java | 29 +-
.../plan/relational/sql/rewrite/ShowRewrite.java | 26 +-
.../sql/rewrite/StatementRewriteFactory.java | 6 +-
.../plan/relational/sql/util/SqlFormatter.java | 42 ++-
.../plan/scheduler/load/LoadTsFileScheduler.java | 117 +++++---
.../rescon/MemSchemaEngineStatistics.java | 4 +-
.../db/storageengine/buffer/BloomFilterCache.java | 2 +-
.../iotdb/db/storageengine/buffer/ChunkCache.java | 2 +-
.../buffer/TimeSeriesMetadataCache.java | 2 +-
.../execute/task/CrossSpaceCompactionTask.java | 2 +-
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../task/RepairUnsortedFileCompactionTask.java | 2 +-
.../compaction/execute/utils/CompactionUtils.java | 12 +-
.../dataregion/memtable/AbstractMemTable.java | 9 -
.../memtable/AlignedReadOnlyMemChunk.java | 318 +++++++--------------
.../memtable/AlignedWritableMemChunk.java | 134 +++------
.../memtable/AlignedWritableMemChunkGroup.java | 5 -
.../dataregion/memtable/IMemTable.java | 2 -
.../memtable/IWritableMemChunkGroup.java | 2 -
.../dataregion/memtable/ReadOnlyMemChunk.java | 16 +-
.../dataregion/memtable/TsFileProcessor.java | 12 +-
.../dataregion/memtable/WritableMemChunk.java | 4 +-
.../dataregion/memtable/WritableMemChunkGroup.java | 8 -
.../read/reader/chunk/MemAlignedChunkReader.java | 188 ++++--------
.../read/reader/chunk/MemChunkReader.java | 4 +-
.../read/reader/chunk/MemPageReader.java | 6 +-
.../dataregion/tsfile/TsFileResource.java | 19 ++
.../dataregion/wal/buffer/IWALBuffer.java | 10 +
.../dataregion/wal/buffer/WALBuffer.java | 13 +
.../storageengine/dataregion/wal/node/WALNode.java | 8 +-
.../memory/LoadTsFileDataCacheMemoryBlock.java | 2 +-
.../load/memory/LoadTsFileMemoryManager.java | 2 +-
.../rescon/memory/PrimitiveArrayManager.java | 2 +-
.../db/storageengine/rescon/memory/SystemInfo.java | 6 +-
.../rescon/memory/TimePartitionManager.java | 5 +-
.../rescon/memory/TsFileResourceManager.java | 4 +-
.../db/utils/datastructure/AlignedTVList.java | 55 ++--
.../datastructure/FixedPriorityBlockingQueue.java | 2 +-
.../MergeSortAlignedTVListIterator.java | 147 +++-------
...tIterator.java => MergeSortTVListIterator.java} | 63 ++--
.../iotdb/db/utils/datastructure/TVList.java | 4 +-
.../pipe/event/TsFileInsertionEventParserTest.java | 1 -
.../plan/analyze/FakePartitionFetcherImpl.java | 5 +
.../plan/planner/distribution/Util.java | 5 +
.../plan/planner/distribution/Util2.java | 5 +
.../plan/relational/analyzer/AnalyzerTest.java | 4 +-
.../plan/relational/analyzer/TSBSMetadata.java | 7 +
.../plan/relational/analyzer/TestMatadata.java | 7 +
.../plan/relational/planner/PlanTester.java | 9 +-
.../relational/sql/parser/AuthorStatementTest.java | 5 -
.../CompactionTableModelTestFileWriter.java | 14 +
.../read/reader/chunk/MemChunkLoaderTest.java | 26 +-
.../iotdb/commons/auth/entity/PrivilegeType.java | 2 +-
.../commons/memory/AtomicLongMemoryBlock.java | 2 +-
.../apache/iotdb/commons/memory/IMemoryBlock.java | 13 +-
.../apache/iotdb/commons/memory/MemoryManager.java | 146 +++++-----
.../schema/column/ColumnHeaderConstant.java | 7 +-
.../table/AlterOrDropTableOperationType.java | 8 +-
.../commons/schema/table/InformationSchema.java | 6 +
.../apache/iotdb/commons/schema/table/TsTable.java | 1 +
.../iotdb/commons/memory/MemoryManagerTest.java | 14 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 19 +-
.../src/main/thrift/confignode.thrift | 1 +
168 files changed, 2775 insertions(+), 1627 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 6840e00537a,cff92d4d359..23441e86b20
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@@ -2602,8 -2602,8 +2602,8 @@@ public class IoTDBDescriptor
long newSize =
storageEngineMemoryManager.getTotalMemorySizeInBytes()
+ consensusMemoryManager.getTotalMemorySizeInBytes();
- consensusMemoryManager.setTotalAllocatedMemorySizeInBytes(0);
+ consensusMemoryManager.setTotalMemorySizeInBytes(0);
- storageEngineMemoryManager.setTotalMemorySizeInBytesWithReload(newSize);
+ storageEngineMemoryManager.setTotalAllocatedMemorySizeInBytesWithReload(newSize);
SystemInfo.getInstance().loadWriteMemory();
}
diff --cc iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
index 2f55481a06e,03fb10d393a..489116f6848
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/memory/MemoryManager.java
@@@ -44,19 -44,13 +44,19 @@@ public class MemoryManager
private final String name;
/** Whether memory management is enabled */
- private final boolean enable;
+ private final boolean enabled;
+ /** The total allocate memory size in byte of memory manager */
- private long totalAllocatedMemorySizeInBytes;
++ private volatile long totalAllocatedMemorySizeInBytes;
+
/** The total memory size in byte of memory manager */
- private long totalMemorySizeInBytes;
+ private volatile long totalMemorySizeInBytes;
+ /** The static allocated memory size */
- private long staticAllocatedMemorySizeInBytes = 0L;
++ private volatile long staticAllocatedMemorySizeInBytes = 0L;
+
/** The allocated memory size */
- private long allocatedMemorySizeInBytes = 0L;
+ private volatile long allocatedMemorySizeInBytes = 0L;
/** The parent memory manager */
private final MemoryManager parentMemoryManager;
@@@ -71,17 -65,15 +71,17 @@@
public MemoryManager(long totalMemorySizeInBytes) {
this.name = "Test";
this.parentMemoryManager = null;
+ this.totalAllocatedMemorySizeInBytes = totalMemorySizeInBytes;
this.totalMemorySizeInBytes = totalMemorySizeInBytes;
- this.enable = false;
+ this.enabled = false;
}
MemoryManager(String name, MemoryManager parentMemoryManager, long totalMemorySizeInBytes) {
this.name = name;
this.parentMemoryManager = parentMemoryManager;
+ this.totalAllocatedMemorySizeInBytes = totalMemorySizeInBytes;
this.totalMemorySizeInBytes = totalMemorySizeInBytes;
- this.enable = false;
+ this.enabled = false;
}
private MemoryManager(
@@@ -250,12 -248,16 +256,19 @@@
name,
(blockName, block) -> {
if (block != null) {
- LOGGER.warn("register memory block already exists: {}", block);
+ if (block.getTotalMemorySizeInBytes() != sizeInBytes) {
+ LOGGER.warn(
+ "getOrRegisterMemoryBlock failed: memory block {} already exists, "
+ + "it's size is {}, requested size is {}",
+ blockName,
+ block.getTotalMemorySizeInBytes(),
+ sizeInBytes);
+ }
return block;
} else {
+ if (type.equals(MemoryBlockType.STATIC)) {
+ staticAllocatedMemorySizeInBytes += sizeInBytes;
+ }
allocatedMemorySizeInBytes += sizeInBytes;
return new AtomicLongMemoryBlock(name, this, sizeInBytes, type);
}
@@@ -458,28 -458,12 +472,32 @@@
return totalMemorySizeInBytes;
}
+ public void setTotalMemorySizeInBytes(long totalMemorySizeInBytes) {
+ this.totalMemorySizeInBytes = totalMemorySizeInBytes;
+ }
+
- public void setTotalMemorySizeInBytesWithReload(long totalMemorySizeInBytes) {
- reAllocateMemoryAccordingToRatio((double) totalMemorySizeInBytes / this.totalMemorySizeInBytes);
+ public void setTotalAllocatedMemorySizeInBytes(long totalAllocatedMemorySizeInBytes) {
+ this.totalMemorySizeInBytes +=
+ (totalAllocatedMemorySizeInBytes - this.totalAllocatedMemorySizeInBytes);
+ this.totalAllocatedMemorySizeInBytes = totalAllocatedMemorySizeInBytes;
+ }
+
+ public void setTotalAllocatedMemorySizeInBytesWithReload(long totalAllocatedMemorySizeInBytes) {
+ long beforeTotalMemorySizeInBytes = this.totalMemorySizeInBytes;
+ long afterTotalMemorySizeInBytes =
+ this.totalMemorySizeInBytes
+ + totalAllocatedMemorySizeInBytes
+ - this.totalAllocatedMemorySizeInBytes;
+ reAllocateMemoryAccordingToRatio(
+ (double) afterTotalMemorySizeInBytes / beforeTotalMemorySizeInBytes);
+ }
+
+ public void expandTotalMemorySizeInBytes(long totalMemorySizeInBytes) {
+ this.totalMemorySizeInBytes += totalMemorySizeInBytes;
+ }
+
+ public long getTotalAllocatedMemorySizeInBytes() {
+ return totalAllocatedMemorySizeInBytes;
}
/** Get available memory size in bytes of memory manager */