You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/10/27 08:47:24 UTC
[iotdb] 02/03: Merge branch 'master' into xingtanzjr/migration_add_wait_sync
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/migration_add_wait_sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 31c750172ace3f11eee0f50055bfbae0979a568b
Merge: c5098b4a70 0767935311
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Oct 27 14:13:45 2022 +0800
Merge branch 'master' into xingtanzjr/migration_add_wait_sync
.github/workflows/client-go.yml | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +-
client-cpp/README.md | 41 +-
client-go | 2 +-
client-py/tests/test_dataframe.py | 4 +-
.../client/async/AsyncDataNodeClientPool.java | 8 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +
.../confignode/conf/ConfigNodeDescriptor.java | 6 +
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 3 +-
.../request/read/GetFunctionTablePlan.java | 25 +-
.../request/write/function/CreateFunctionPlan.java | 52 +-
.../consensus/response/FunctionTableResp.java | 70 +++
.../iotdb/confignode/manager/ConfigManager.java | 27 +-
.../iotdb/confignode/manager/ConsensusManager.java | 180 +++---
.../apache/iotdb/confignode/manager/IManager.java | 6 +-
.../iotdb/confignode/manager/UDFManager.java | 122 ++--
.../iotdb/confignode/manager/load/LoadManager.java | 2 +-
.../manager/load/LoadManagerMetrics.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 2 +-
.../iotdb/confignode/persistence/UDFInfo.java | 119 ++--
.../persistence/executor/ConfigPlanExecutor.java | 4 +-
.../persistence/metric/NodeInfoMetrics.java | 4 +-
.../persistence/metric/PartitionInfoMetrics.java | 4 +-
.../persistence/partition/PartitionInfo.java | 6 +-
.../iotdb/confignode/service/ConfigNode.java | 12 +-
.../service/thrift/ConfigNodeRPCService.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandler.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 17 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 9 +
.../resources/confignode1conf/iotdb-metric.yml | 5 -
.../resources/confignode2conf/iotdb-metric.yml | 5 -
.../resources/confignode3conf/iotdb-metric.yml | 5 -
.../org/apache/iotdb/consensus/IConsensus.java | 24 +
.../multileader/MultiLeaderConsensus.java | 12 +
.../multileader/MultiLeaderServerImpl.java | 96 ++++
.../multileader/MultiLeaderServerMetrics.java | 88 +++
.../multileader/client/DispatchLogHandler.java | 15 +
.../multileader/logdispatcher/LogDispatcher.java | 23 +
.../logdispatcher/LogDispatcherThreadMetrics.java | 95 ++++
.../logdispatcher/MultiLeaderMemoryManager.java | 10 +-
.../MultiLeaderMemoryManagerMetrics.java | 41 +-
.../service/MultiLeaderRPCServiceProcessor.java | 31 ++
.../iotdb/consensus/ratis/RatisConsensus.java | 60 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 14 +
docs/UserGuide/API/Interface-Comparison.md | 44 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 6 -
docs/zh/UserGuide/API/Interface-Comparison.md | 44 +-
.../UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 7 -
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 23 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 8 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +-
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 8 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 3 +
.../it/IoTDBSyntaxConventionStringLiteralIT.java | 5 +
.../db/it/aggregation/IoTDBTagAggregationIT.java | 16 +
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 4 +-
.../iotdb/db/it/env/StandaloneEnvConfig.java | 13 +
.../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 59 ++
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 528 ++++++++++++++++++
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 3 +
.../db/it/udf/IoTDBUDFOtherWindowQueryIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java | 2 +
.../db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java | 3 +
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 2 +
.../iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java | 2 +
.../org/apache/iotdb/db/it/utils/TestUtils.java | 10 +
.../src/test/resources/logback-test.xml | 2 +-
.../db/integration/IoTDBRewriteTsFileToolIT.java | 523 ++++++++++++++++-
.../apache/iotdb/session/pool/SessionPoolTest.java | 4 +-
integration/src/test/resources/logback-test.xml | 2 +-
metrics/ReadMe.md | 9 +-
.../resources/conf/iotdb-confignode-metric.yml | 5 -
.../resources/conf/iotdb-datanode-metric.yml | 5 -
.../iotdb/metrics/AbstractMetricService.java | 19 +-
.../iotdb/metrics/DoNothingMetricService.java | 6 -
.../apache/iotdb/metrics/config/MetricConfig.java | 15 -
.../metrics/config/MetricConfigDescriptor.java | 1 -
.../jvm/JvmClassLoaderMetrics.java | 2 +-
.../{predefined => }/jvm/JvmCompileMetrics.java | 2 +-
.../{predefined => }/jvm/JvmGcMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMemoryMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMetrics.java | 2 +-
.../{predefined => }/jvm/JvmThreadMetrics.java | 2 +-
.../metricsets/{predefined => }/jvm/JvmUtils.java | 2 +-
.../{predefined => }/logback/LogbackMetrics.java | 2 +-
.../logback/MetricsTurboFilter.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 1 -
.../interface/src/test/resources/iotdb-metric.yml | 8 -
node-commons/pom.xml | 21 +
.../async/AsyncDataNodeInternalServiceClient.java | 2 +-
.../commons/service/metric}/MetricService.java | 35 +-
.../service/metric}/MetricServiceMBean.java | 2 +-
.../commons/service/metric}/enums/Metric.java | 9 +-
.../commons/service/metric}/enums/Operation.java | 2 +-
.../iotdb/commons/service/metric}/enums/Tag.java | 3 +-
.../apache/iotdb/commons/udf/UDFInformation.java | 83 ++-
.../org/apache/iotdb/commons/udf/UDFTable.java | 57 +-
.../commons/udf/service/UDFManagementService.java | 162 ++++--
.../java/org/apache/iotdb/RewriteTsFileTool.java | 620 +++++++++++++++------
server/pom.xml | 10 -
.../resources/conf/iotdb-datanode.properties | 1 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 330 +++++++++++
.../iotdb/db/client/DataNodeInternalClient.java | 129 +++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../iotdb/db/engine/TsFileMetricManager.java | 7 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../iotdb/db/engine/cache/ChunkCacheMetrics.java | 4 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
.../cache/TimeSeriesMetadataCacheMetrics.java | 4 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 2 +-
.../iotdb/db/engine/flush/FlushManagerMetrics.java | 4 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 6 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 2 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 2 +-
.../storagegroup/TsFileProcessorInfoMetrics.java | 6 +-
.../iotdb/db/exception/IntoProcessException.java | 16 +-
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 4 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 247 ++++----
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 128 +++--
.../db/metadata/rescon/SchemaResourceManager.java | 2 +-
.../rescon/SchemaStatisticsManagerMetrics.java | 4 +-
.../apache/iotdb/db/mpp/common/SessionInfo.java | 19 +
.../execution/exchange/MPPDataExchangeService.java | 2 +-
.../exchange/MPPDataExchangeServiceMetrics.java | 4 +-
.../MPPDataExchangeServiceThriftHandler.java | 2 +-
...MppDataExchangeServiceThriftHandlerMetrics.java | 6 +-
.../fragment/FragmentInstanceContext.java | 29 +-
.../fragment/FragmentInstanceManager.java | 6 +-
.../db/mpp/execution/operator/OperatorContext.java | 5 +
.../operator/process/AbstractIntoOperator.java | 384 +++++++++++++
.../operator/process/DeviceViewIntoOperator.java | 140 +++++
.../execution/operator/process/IntoOperator.java | 104 ++++
.../process/RawDataAggregationOperator.java | 2 +-
.../process/SingleInputAggregationOperator.java | 2 -
.../process/SlidingWindowAggregationOperator.java | 2 +-
.../schema/TimeSeriesSchemaScanOperator.java | 9 +-
.../execution/schedule/AbstractDriverThread.java | 60 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 38 +-
.../mpp/execution/schedule/DriverTaskThread.java | 5 +-
.../schedule/DriverTaskTimeoutSentinelThread.java | 5 +-
.../execution/schedule/ThreadProducer.java} | 19 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 8 +-
.../config/executor/ClusterConfigTaskExecutor.java | 113 +++-
.../config/executor/IConfigTaskExecutor.java | 8 +-
.../executor/StandaloneConfigTaskExecutor.java | 21 +-
.../config/metadata/CreateFunctionTask.java | 15 +-
.../config/metadata/ShowFunctionsTask.java | 155 ++----
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 59 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 40 ++
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 7 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 123 ++++
.../planner/distribution/ExchangeNodeAdder.java | 6 +-
.../SimpleFragmentParallelPlanner.java | 1 +
.../plan/planner/distribution/SourceRewriter.java | 11 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../db/mpp/plan/planner/plan/FragmentInstance.java | 22 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 10 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 10 +
.../metedata/read/SchemaQueryOrderByHeatNode.java | 4 +-
.../planner/plan/node/process/AggregationNode.java | 23 +-
.../planner/plan/node/process/DeviceMergeNode.java | 32 +-
.../{TimeJoinNode.java => DeviceViewIntoNode.java} | 95 ++--
.../planner/plan/node/process/DeviceViewNode.java | 19 +-
.../planner/plan/node/process/ExchangeNode.java | 41 +-
.../plan/planner/plan/node/process/FillNode.java | 46 +-
.../plan/node/process/GroupByLevelNode.java | 29 +-
.../planner/plan/node/process/GroupByTagNode.java | 17 +-
.../node/process/{LimitNode.java => IntoNode.java} | 94 ++--
.../plan/planner/plan/node/process/LimitNode.java | 37 +-
...tiChildNode.java => MultiChildProcessNode.java} | 23 +-
.../plan/planner/plan/node/process/OffsetNode.java | 36 +-
.../planner/plan/node/process/ProjectNode.java | 40 +-
...iChildNode.java => SingleChildProcessNode.java} | 50 +-
.../node/process/SlidingWindowAggregationNode.java | 36 +-
.../plan/planner/plan/node/process/SortNode.java | 29 +-
.../planner/plan/node/process/TimeJoinNode.java | 17 +-
.../planner/plan/node/process/TransformNode.java | 29 +-
.../node/process/last/LastQueryCollectNode.java | 4 +-
.../plan/node/process/last/LastQueryMergeNode.java | 4 +-
.../plan/node/process/last/LastQueryNode.java | 4 +-
.../parameter/DeviceViewIntoPathDescriptor.java | 11 +-
.../planner/plan/parameter/IntoPathDescriptor.java | 10 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 6 +-
.../plan/statement/component/IntoComponent.java | 8 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 19 +-
.../metadata/CreateFunctionStatement.java | 21 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 22 +-
.../db/query/pool/RawQueryReadTaskPoolManager.java | 2 +-
.../pool/RawQueryReadTaskPoolManagerMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 128 ++++-
.../db/service/DataNodeInternalRPCService.java | 2 +-
.../service/DataNodeInternalRPCServiceMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 5 +-
.../org/apache/iotdb/db/service/RPCService.java | 2 +-
.../apache/iotdb/db/service/RPCServiceMetrics.java | 4 +-
.../db/service/ResourcesInformationHolder.java | 12 +
...erviceMBean.java => DataNodeMetricsHelper.java} | 25 +-
.../metrics/{predefined => }/FileMetrics.java | 6 +-
.../metrics/{predefined => }/ProcessMetrics.java | 6 +-
.../metrics/{predefined => }/SystemMetrics.java | 6 +-
.../metrics/recorder/CacheMetricsRecorder.java | 6 +-
.../recorder/CompactionMetricsRecorder.java | 6 +-
.../db/service/thrift/ProcessorWithMetrics.java | 6 +-
.../handler/InternalServiceThriftHandler.java | 2 +-
.../InternalServiceThriftHandlerMetrics.java | 4 +-
.../thrift/handler/RPCServiceThriftHandler.java | 2 +-
.../handler/RPCServiceThriftHandlerMetrics.java | 4 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 32 +-
.../db/service/thrift/impl/TSServiceImpl.java | 4 +-
.../trigger/service/TriggerManagementService.java | 18 +-
.../DriverTaskTimeoutSentinelThreadTest.java | 23 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeFailTest.java | 87 ++-
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 8 +-
.../mpp/plan/plan/FragmentInstanceSerdeTest.java | 10 +-
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 40 ++
.../plan/plan/node/process/IntoNodeSerdeTest.java | 83 +++
.../plan/scheduler/StandaloneSchedulerTest.java | 18 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 8 +-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 2 +
.../test/resources/datanode1conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode2conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode3conf/iotdb-metric.yml | 7 +-
server/src/test/resources/logback-test.xml | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 5 +
.../java/org/apache/iotdb/session/ISession.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 258 ++++++---
.../org/apache/iotdb/session/SessionConfig.java | 2 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 48 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 4 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 2 +-
.../java/org/apache/iotdb/db/sql/ClusterIT.java | 2 +-
.../src/main/thrift/confignode.thrift | 47 +-
.../src/main/thrift/mutlileader.thrift | 10 +
thrift/src/main/thrift/datanode.thrift | 16 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 72 ++-
.../java/org/apache/iotdb/tsfile/utils/BitMap.java | 6 +
250 files changed, 6014 insertions(+), 1972 deletions(-)
diff --cc consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
index 5e2cb5fb7e,911b79c232..dc4ee7007b
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
@@@ -45,10 -50,9 +50,11 @@@ import org.apache.iotdb.consensus.multi
import org.apache.iotdb.consensus.multileader.thrift.TSendSnapshotFragmentRes;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadReq;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadRes;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteReq;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteRes;
import org.apache.iotdb.consensus.multileader.wal.ConsensusReqReader;
import org.apache.iotdb.consensus.multileader.wal.GetConsensusReqReaderPlan;
+ import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
diff --cc thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
index 81252ce353,f85c79c625..0336c19303
--- a/thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
+++ b/thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
@@@ -110,7 -109,7 +119,8 @@@ service MultiLeaderConsensusIService
TActivatePeerRes activatePeer(TActivatePeerReq req)
TBuildSyncLogChannelRes buildSyncLogChannel(TBuildSyncLogChannelReq req)
TRemoveSyncLogChannelRes removeSyncLogChannel(TRemoveSyncLogChannelReq req)
+ TWaitSyncLogCompleteRes waitSyncLogComplete(TWaitSyncLogCompleteReq req)
TSendSnapshotFragmentRes sendSnapshotFragment(TSendSnapshotFragmentReq req)
TTriggerSnapshotLoadRes triggerSnapshotLoad(TTriggerSnapshotLoadReq req)
+ TCleanupTransferredSnapshotRes cleanupTransferredSnapshot(TCleanupTransferredSnapshotReq req)
}