You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2025/08/25 02:31:06 UTC
(iotdb) 01/01: Merge branch 'iotdb_master' into optimizeMemtableScan
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch optimizeMemtableScan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 05d578231cab987c4cf0d92631390c46efb7c326
Merge: de65501fd57 72a11f37168
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Mon Aug 25 10:30:12 2025 +0800
Merge branch 'iotdb_master' into optimizeMemtableScan
LICENSE-binary | 59 +--
NOTICE | 2 +-
NOTICE-binary | 2 +-
.../org/apache/iotdb/it/utils/TsFileGenerator.java | 28 ++
.../iotdb/it/utils/TsFileTableGenerator.java | 17 +-
.../IoTDBRegionGroupExpandAndShrinkForIoTV1IT.java | 329 ++++++++++++++-
.../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 155 +------
.../org/apache/iotdb/db/it/IoTDBRestartIT.java | 59 +++
.../treemodel/auto/basic/IoTDBPipeDataSinkIT.java | 13 +-
.../relational/it/db/it/IoTDBLoadTsFileIT.java | 299 ++++++++++++++
.../it/insertquery/IoTDBInsertQueryIT.java | 12 +-
.../it/query/recent/IoTDBUnionTableIT.java | 127 ++++++
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 180 ++++++++-
.../org/apache/iotdb/tool/common/Constants.java | 2 +
.../iotdb/tool/schema/ExportSchemaTable.java | 44 +-
iotdb-core/ainode/ainode/core/constant.py | 6 -
.../{scheduler => dispatcher}/__init__.py | 0
.../dispatcher/abstract_dispatcher.py} | 31 +-
.../core/inference/dispatcher/basic_dispatcher.py | 60 +++
.../ainode/core/inference/inference_request.py | 2 +
.../core/inference/inference_request_pool.py | 19 +-
.../ainode/core/inference/pool_controller.py | 188 +++++++++
.../ainode/ainode/core/inference/pool_group.py | 93 +++++
.../{scheduler => pool_scheduler}/__init__.py | 0
.../pool_scheduler/abstract_pool_scheduler.py | 56 +++
.../pool_scheduler/basic_pool_scheduler.py | 59 +++
.../{scheduler => request_scheduler}/__init__.py | 0
.../abstract_request_scheduler.py} | 2 +-
.../basic_request_scheduler.py} | 9 +-
iotdb-core/ainode/ainode/core/log.py | 13 +
.../ainode/core/manager/inference_manager.py | 134 ++----
iotdb-core/ainode/ainode/core/manager/utils.py | 8 +-
iotdb-core/ainode/ainode/core/util/decorator.py | 15 +
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +
.../confignode/conf/ConfigNodeDescriptor.java | 6 +
.../exception/NoAvailableRegionGroupException.java | 14 +-
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../iotdb/confignode/manager/ProcedureManager.java | 63 ++-
.../manager/consensus/ConsensusManager.java | 5 +
.../manager/partition/PartitionManager.java | 4 +-
.../persistence/schema/TemplateTable.java | 2 +-
.../apache/iotdb/consensus/config/RatisConfig.java | 27 +-
.../exception/RatisRequestFailedException.java | 9 +
.../iotdb/consensus/ratis/RatisConsensus.java | 23 +-
iotdb-core/datanode/pom.xml | 4 -
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 21 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++
.../db/consensus/DataRegionConsensusImpl.java | 4 +
.../db/consensus/SchemaRegionConsensusImpl.java | 5 +
.../tsfile/parser/TsFileInsertionEventParser.java | 11 +-
.../query/TsFileInsertionEventQueryParser.java | 215 +++++-----
...ileInsertionEventQueryParserTabletIterator.java | 14 +-
.../scan/TsFileInsertionEventScanParser.java | 184 +++++----
.../table/TsFileInsertionEventTableParser.java | 256 ++++++------
...ileInsertionEventTableParserTabletIterator.java | 79 ++--
.../processor/aggregate/AggregateProcessor.java | 5 +-
.../pipeconsensus/PipeConsensusReceiver.java | 114 +++---
...istoricalDataRegionTsFileAndDeletionSource.java | 11 +-
.../realtime/PipeRealtimeDataRegionSource.java | 14 +-
.../realtime/assigner/PipeDataRegionAssigner.java | 16 +-
.../db/pipe/source/mqtt/MQTTPublishHandler.java | 21 +-
.../iotdb/db/pipe/source/mqtt/MQTTSource.java | 102 ++++-
.../protocol/rest/v2/impl/RestApiServiceImpl.java | 43 +-
.../protocol/thrift/impl/ClientRPCServiceImpl.java | 13 +-
.../db/queryengine/common/MPPQueryContext.java | 7 +-
.../queryengine/execution/driver/DataDriver.java | 44 +-
.../fragment/FakedFragmentInstanceContext.java | 5 +-
.../fragment/FragmentInstanceContext.java | 244 +++++++----
.../fragment/FragmentInstanceExecution.java | 8 +-
.../execution/fragment/QueryContext.java | 5 +-
.../queryengine/metric/QueryResourceMetricSet.java | 26 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 69 +++-
.../queryengine/plan/analyze/PredicateUtils.java | 11 +-
.../analyze/load/LoadTsFileTableSchemaCache.java | 19 +-
.../analyze/load/LoadTsFileTreeSchemaCache.java | 24 +-
.../load/TreeSchemaAutoCreatorAndVerifier.java | 2 +-
.../analyze/schema/AutoCreateSchemaExecutor.java | 13 +-
.../plan/analyze/schema/TemplateSchemaFetcher.java | 2 +-
.../config/executor/ClusterConfigTaskExecutor.java | 4 +-
.../config/metadata/region/ExtendRegionTask.java | 2 +-
.../config/metadata/region/RemoveRegionTask.java | 2 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 19 +-
.../plan/planner/LocalExecutionPlanner.java | 7 +-
.../plan/planner/TableOperatorGenerator.java | 22 +-
.../plan/AbstractFragmentParallelPlanner.java | 20 +-
.../plan/planner/plan/node/PlanGraphPrinter.java | 9 +
.../plan/planner/plan/node/PlanNodeType.java | 4 +
.../plan/planner/plan/node/PlanVisitor.java | 5 +
.../relational/analyzer/StatementAnalyzer.java | 6 +-
.../fetcher/TableHeaderSchemaValidator.java | 4 +-
.../plan/relational/planner/QueryPlanner.java | 34 ++
.../plan/relational/planner/RelationPlanner.java | 92 ++++-
.../relational/planner/TableLogicalPlanner.java | 5 +-
.../distribute/TableDistributedPlanGenerator.java | 7 +
.../iterative/rule/PruneDistinctAggregation.java | 6 +
.../planner/iterative/rule/PruneUnionColumns.java | 80 ++++
.../iterative/rule/PruneUnionSourceColumns.java | 52 +++
.../plan/relational/planner/node/Patterns.java | 4 +
.../relational/planner/node/SetOperationNode.java | 140 +++++++
.../plan/relational/planner/node/UnionNode.java | 97 +++++
.../optimizations/LogicalOptimizeFactory.java | 6 +-
.../optimizations/PushPredicateIntoTableScan.java | 32 +-
.../TransformAggregationToStreamable.java | 6 +
.../optimizations/TransformSortToStreamSort.java | 7 +
.../optimizations/UnaliasSymbolReferences.java | 54 +++
.../plan/relational/sql/ast/ExtendRegion.java | 16 +-
.../plan/relational/sql/ast/RemoveRegion.java | 16 +-
.../relational/sql/ast/WrappedInsertStatement.java | 2 +-
.../plan/relational/sql/parser/AstBuilder.java | 10 +-
.../plan/relational/type/CompatibleResolver.java | 108 +++++
.../metadata/region/ExtendRegionStatement.java | 10 +-
.../metadata/region/RemoveRegionStatement.java | 10 +-
.../FragmentInstanceStatisticsDrawer.java | 11 +
.../metrics/IoTDBInternalLocalReporter.java | 3 +-
.../db/storageengine/dataregion/DataRegion.java | 450 ++++++++++++++-------
.../dataregion/IDataRegionForQuery.java | 29 +-
.../dataregion/VirtualDataRegion.java | 36 +-
.../performer/impl/FastCompactionPerformer.java | 5 +-
.../compaction/execute/utils/CompactionUtils.java | 36 ++
.../execute/utils/MultiTsFileDeviceIterator.java | 45 +--
.../FirstBatchCompactionAlignedChunkWriter.java | 3 +-
.../executor/fast/SeriesCompactionExecutor.java | 19 +-
.../selector/impl/SettleSelectorImpl.java | 23 +-
.../memtable/AbstractWritableMemChunk.java | 11 +
.../dataregion/memtable/TsFileProcessor.java | 65 ++-
.../dataregion/tsfile/TsFileManager.java | 43 +-
.../db/storageengine/load/LoadTsFileManager.java | 22 +-
.../memory/LoadTsFileDataCacheMemoryBlock.java | 8 +-
.../load/memory/LoadTsFileMemoryManager.java | 15 +-
.../apache/iotdb/db/utils/ModificationUtils.java | 59 +--
.../db/queryengine/execution/DataDriverTest.java | 15 +-
.../plan/relational/analyzer/UnionTest.java | 65 +++
.../InformationSchemaTableScanMatcher.java | 2 +-
.../planner/assertions/PlanMatchPattern.java | 5 +
.../tablemodel/CompactionWithAllNullRowsTest.java | 73 +++-
iotdb-core/metrics/interface/pom.xml | 1 -
.../conf/iotdb-system.properties.template | 16 +-
.../commons/auth/authorizer/OpenIdAuthorizer.java | 10 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 40 +-
.../iotdb/commons/enums/ReadConsistencyLevel.java} | 15 +-
.../apache/iotdb/commons/path/PatternTreeMap.java | 4 +
.../iotdb/commons/pipe/config/PipeConfig.java | 8 +-
.../iotdb/commons/pipe/config/PipeDescriptor.java | 8 +-
.../pipe/config/constant/PipeSourceConstant.java | 3 +-
.../commons/pipe/event/ProgressReportEvent.java | 33 +-
.../pipe/source/IoTDBNonDataRegionSource.java | 11 +-
.../db/relational/grammar/sql/RelationalSql.g4 | 6 +-
.../src/main/thrift/confignode.thrift | 6 +-
.../src/main/thrift/datanode.thrift | 1 +
pom.xml | 12 +-
151 files changed, 4534 insertions(+), 1413 deletions(-)
diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
index 165a218a9b5,95d90dfa63d..351e64650a7
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ModificationUtils.java
@@@ -20,7 -20,9 +20,10 @@@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.exception.IllegalPathException;
+ import org.apache.iotdb.commons.path.AlignedPath;
+ import org.apache.iotdb.commons.path.PatternTreeMap;
+import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering;
+ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.selector.impl.SettleSelectorImpl;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;