You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ca...@apache.org on 2022/11/24 11:35:52 UTC
[iotdb] branch beyyes/fix_remove_node_problem_1123 created (now 822f4f1c2f)
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 822f4f1c2f perfect the migrate process
This branch includes the following new commits:
new 887a4d42f1 Merge commit 'refs/pull/8091/head' of github.com:apache/iotdb
new 65b6eee421 merge master
new f50a871124 Merge branch 'master' of github.com:apache/iotdb
new 90c81634e1 Merge branch 'master' of github.com:apache/iotdb
new 822f4f1c2f perfect the migrate process
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 02/05: merge master
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 65b6eee4212c59978e7d30a8039a499bde9b0d20
Merge: 887a4d42f1 03c826930a
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Wed Nov 23 23:23:26 2022 +0800
merge master
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 20 +-
confignode/src/assembly/confignode.xml | 4 -
.../resources/conf/iotdb-confignode.properties | 68 ++-
.../confignode/client/ConfigNodeRequestType.java | 2 +-
.../client/sync/SyncConfigNodeClientPool.java | 5 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 51 +-
.../confignode/conf/ConfigNodeDescriptor.java | 38 +-
.../confignode/conf/ConfigNodeStartupCheck.java | 23 +-
.../consensus/request/ConfigPhysicalPlan.java | 6 +-
.../consensus/request/ConfigPhysicalPlanType.java | 2 +-
.../write/region/CreateRegionGroupsPlan.java | 19 +
...tPlan.java => AdjustMaxRegionGroupNumPlan.java} | 46 +-
.../confignode/manager/ClusterSchemaManager.java | 80 +--
.../iotdb/confignode/manager/ConfigManager.java | 9 +-
.../iotdb/confignode/manager/ConsensusManager.java | 52 +-
.../manager/load/balancer/RegionBalancer.java | 1 +
.../manager/load/balancer/RouteBalancer.java | 117 ++--
.../manager/load/balancer/router/LeaderRouter.java | 90 ----
.../router/leader/GreedyLeaderBalancer.java | 186 +++++++
.../{IRouter.java => leader/ILeaderBalancer.java} | 30 +-
.../MinCostFlowLeaderBalancer.java} | 100 +++-
.../manager/load/balancer/router/mcf/MCFEdge.java | 34 --
.../GreedyPriorityBalancer.java} | 65 ++-
.../IPriorityBalancer.java} | 24 +-
.../LeaderPriorityBalancer.java} | 49 +-
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 60 ++-
.../persistence/executor/ConfigPlanExecutor.java | 6 +-
.../persistence/partition/PartitionInfo.java | 7 +-
.../persistence/schema/ClusterSchemaInfo.java | 20 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 73 ++-
.../procedure/env/DataNodeRemoveHandler.java | 1 -
.../impl/node/AddConfigNodeProcedure.java | 38 +-
.../impl/schema/DeactivateTemplateProcedure.java | 6 +-
.../impl/schema/DeleteTimeSeriesProcedure.java | 6 +-
.../impl/schema/UnsetTemplateProcedure.java | 4 +-
.../statemachine/CreateRegionGroupsProcedure.java | 13 +-
.../procedure/state/AddConfigNodeState.java | 3 +-
.../iotdb/confignode/service/ConfigNode.java | 7 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 12 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 8 +-
.../router/leader/GreedyLeaderBalancerTest.java | 146 +++++
.../leader/LeaderBalancerComparisonTest.java | 369 +++++++++++++
.../MinCostFlowLeaderBalancerTest.java} | 33 +-
.../GreedyPriorityTest.java} | 8 +-
.../LeaderPriorityBalancerTest.java} | 56 +-
.../confignode1conf/iotdb-confignode-metric.yml | 48 --
.../confignode1conf/iotdb-confignode.properties | 10 +-
.../confignode2conf/iotdb-confignode-metric.yml | 48 --
.../confignode2conf/iotdb-confignode.properties | 10 +-
.../confignode3conf/iotdb-confignode-metric.yml | 48 --
.../confignode3conf/iotdb-confignode.properties | 10 +-
.../iotdb/consensus/ratis/RatisConsensus.java | 44 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 23 +-
distribution/src/assembly/all.xml | 12 -
distribution/src/assembly/confignode.xml | 6 -
distribution/src/assembly/datanode.xml | 8 -
docs/UserGuide/Cluster/Cluster-Concept.md | 4 +-
docs/UserGuide/Cluster/Cluster-Setup.md | 66 +--
docs/UserGuide/Data-Concept/Deadband-Process.md | 2 +-
docs/UserGuide/Data-Concept/Schema-Template.md | 15 +
.../{Write-Data => Maintenance-Tools}/CSV-Tool.md | 0
.../Maintenance-Tools/Maintenance-Command.md | 126 ++---
docs/UserGuide/Maintenance-Tools/NodeTool.md | 297 -----------
.../TsFile-Load-Export-Tool.md} | 0
docs/UserGuide/Monitor-Alert/Metric-Tool.md | 68 +--
.../{Storage-Group.md => Database.md} | 4 +-
docs/UserGuide/Operate-Metadata/Template.md | 2 +-
docs/UserGuide/Operate-Metadata/Timeseries.md | 4 +-
docs/UserGuide/Reference/Keywords.md | 401 +++++++-------
docs/UserGuide/Reference/SQL-Reference.md | 12 +-
docs/UserGuide/Reference/Syntax-Conventions.md | 587 ---------------------
.../Syntax-Conventions/Detailed-Grammar.md | 28 +
docs/UserGuide/Syntax-Conventions/Identifier.md | 141 +++++
docs/UserGuide/Syntax-Conventions/KeyValue-Pair.md | 119 +++++
.../Keywords-And-Reserved-Words.md | 26 +
.../UserGuide/Syntax-Conventions/Literal-Values.md | 157 ++++++
.../Syntax-Conventions/NodeName-In-Path.md | 119 +++++
.../Syntax-Conventions/Session-And-TsFile-API.md | 119 +++++
docs/UserGuide/Write-Data/Batch-Load-Tool.md | 32 ++
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 49 +-
docs/zh/UserGuide/Data-Concept/Deadband-Process.md | 2 +-
docs/zh/UserGuide/Data-Concept/Schema-Template.md | 15 +
.../{Write-Data => Maintenance-Tools}/CSV-Tool.md | 2 +-
.../Maintenance-Tools/Maintenance-Command.md | 120 ++---
docs/zh/UserGuide/Maintenance-Tools/NodeTool.md | 293 ----------
.../TsFile-Load-Export-Tool.md} | 0
docs/zh/UserGuide/Monitor-Alert/Metric-Tool.md | 67 +--
.../{Storage-Group.md => Database.md} | 6 +-
docs/zh/UserGuide/Operate-Metadata/Template.md | 4 +-
docs/zh/UserGuide/Operate-Metadata/Timeseries.md | 14 +-
docs/zh/UserGuide/Reference/Keywords.md | 401 +++++++-------
docs/zh/UserGuide/Reference/SQL-Reference.md | 12 +-
docs/zh/UserGuide/Reference/Syntax-Conventions.md | 586 --------------------
.../Syntax-Conventions/Detailed-Grammar.md | 28 +
docs/zh/UserGuide/Syntax-Conventions/Identifier.md | 142 +++++
.../UserGuide/Syntax-Conventions/KeyValue-Pair.md | 119 +++++
.../Keywords-And-Reserved-Words.md | 26 +
.../UserGuide/Syntax-Conventions/Literal-Values.md | 150 ++++++
.../Syntax-Conventions/NodeName-In-Path.md | 120 +++++
.../Syntax-Conventions/Session-And-TsFile-API.md | 119 +++++
docs/zh/UserGuide/Write-Data/Batch-Load-Tool.md | 32 ++
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 9 +
.../java/org/apache/iotdb/it/env/MppConfig.java | 7 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../it/IoTDBClusterRegionLeaderBalancingIT.java | 26 +-
.../iotdb/confignode/it/IoTDBClusterRestartIT.java | 8 +-
.../it/partition/IoTDBPartitionGetterIT.java | 55 +-
.../partition/IoTDBPartitionInheritPolicyTest.java | 3 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 2 +-
.../db/it/alignbydevice/IoTDBAlignByDeviceIT.java | 75 +--
.../db/it/aligned/IoTDBGroupByLevelQueryIT.java | 39 +-
.../db/it/schema/IoTDBCreateTimeseriesIT.java | 2 +-
.../db/it/schema/IoTDBDeactivateTemplateIT.java | 8 +-
.../iotdb/db/it/schema/IoTDBMetadataFetchIT.java | 2 +-
.../iotdb/db/it/schema/IoTDBSchemaTemplateIT.java | 4 +-
.../db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java | 20 +
metrics/ReadMe.md | 29 +-
.../reporter/DropwizardPrometheusReporter.java | 2 +-
.../resources/conf/iotdb-confignode-metric.yml | 49 --
.../resources/conf/iotdb-datanode-metric.yml | 49 --
.../iotdb/metrics/AbstractMetricService.java | 6 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 41 +-
.../metrics/config/MetricConfigDescriptor.java | 186 ++++---
.../iotdb/metrics/config/MetricConstant.java | 37 --
.../{MonitorType.java => MetricFrameType.java} | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 79 +--
.../interface/src/test/resources/iotdb-metric.yml | 52 --
.../reporter/MicrometerPrometheusReporter.java | 4 +-
.../resources/conf/iotdb-common.properties | 22 +-
.../apache/iotdb/commons/conf/IoTDBConstant.java | 16 +
.../schemaregion/rocksdb/RSchemaRegion.java | 33 +-
.../metadata/tagSchemaRegion/TagSchemaRegion.java | 18 +-
.../resources/conf/iotdb-datanode.properties | 65 +++
.../assembly/resources/tools/rocksdb-transfer.bat | 126 -----
.../assembly/resources/tools/rocksdb-transfer.sh | 82 ---
server/src/assembly/server.xml | 4 -
.../apache/iotdb/db/client/ConfigNodeClient.java | 5 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +-
.../iotdb/db/metadata/LocalSchemaProcessor.java | 18 -
.../iotdb/db/metadata/mtree/ConfigMTree.java | 2 +-
.../iotdb/db/metadata/mtree/IMTreeBelowSG.java | 20 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 20 +-
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 20 +-
.../store/disk/schemafile/AliasIndexPage.java | 2 +-
.../mtree/store/disk/schemafile/InternalPage.java | 2 +-
.../mtree/store/disk/schemafile/SchemaFile.java | 40 +-
.../disk/schemafile/pagemgr/IPageManager.java | 5 +-
.../store/disk/schemafile/pagemgr/PageManager.java | 10 +-
.../mtree/traverser/counter/CounterTraverser.java | 4 +-
.../counter/MeasurementGroupByLevelCounter.java | 12 +-
.../db/metadata/schemaregion/ISchemaRegion.java | 19 +-
.../schemaregion/SchemaRegionMemoryImpl.java | 29 +-
.../schemaregion/SchemaRegionSchemaFileImpl.java | 27 +-
.../metadata/visitor/SchemaExecutionVisitor.java | 4 +-
.../db/mpp/common/header/ColumnHeaderConstant.java | 8 +-
.../iotdb/db/mpp/execution/driver/DataDriver.java | 4 +-
.../operator/process/AbstractIntoOperator.java | 21 +-
.../operator/process/DeviceViewIntoOperator.java | 2 +-
.../operator/process/DeviceViewOperator.java | 16 +-
.../execution/operator/process/IntoOperator.java | 2 +-
.../operator/schema/CountMergeOperator.java | 18 +-
.../operator/schema/DevicesCountOperator.java | 4 +-
.../schema/LevelTimeSeriesCountOperator.java | 4 +-
.../operator/schema/NodePathsCountOperator.java | 2 +-
.../operator/schema/TimeSeriesCountOperator.java | 4 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 62 ++-
.../memory/StatementMemorySourceVisitor.java | 6 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 25 +-
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 19 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 28 +-
.../recorder/CompactionMetricsRecorder.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 41 +-
.../db/tools/schema/SchemaFileSketchTool.java | 3 +-
.../iotdb/db/metadata/mtree/MTreeBelowSGTest.java | 27 +-
.../metadata/mtree/schemafile/SchemaFileTest.java | 3 +-
.../apache/iotdb/db/metric/MetricServiceTest.java | 6 +-
.../operator/DeviceMergeOperatorTest.java | 9 +
.../execution/operator/DeviceViewOperatorTest.java | 3 +
.../operator/schema/CountMergeOperatorTest.java | 8 +-
.../operator/schema/SchemaCountOperatorTest.java | 10 +-
.../iotdb/db/tools/SchemaFileSketchTest.java | 14 +-
.../datanode1conf/iotdb-datanode-metric.yml | 48 --
.../datanode1conf/iotdb-datanode.properties | 8 +
.../datanode2conf/iotdb-datanode-metric.yml | 48 --
.../datanode2conf/iotdb-datanode.properties | 8 +
.../datanode3conf/iotdb-datanode-metric.yml | 48 --
.../datanode3conf/iotdb-datanode.properties | 10 +-
.../java/org/apache/iotdb/session/Session.java | 9 +-
site/src/main/.vuepress/config.js | 36 +-
thrift-commons/src/main/thrift/common.thrift | 1 +
.../src/main/thrift/confignode.thrift | 8 +-
thrift/src/main/thrift/client.thrift | 1 +
thrift/src/main/thrift/datanode.thrift | 8 +-
.../iotdb/udf/api/collector/PointCollector.java | 4 +-
195 files changed, 4483 insertions(+), 4421 deletions(-)
diff --cc docs/UserGuide/Cluster/Cluster-Setup.md
index 98c8539321,edeece8026..16b61c6cb9
--- a/docs/UserGuide/Cluster/Cluster-Setup.md
+++ b/docs/UserGuide/Cluster/Cluster-Setup.md
@@@ -127,9 -129,9 +129,9 @@@ The adding ConfigNode also use the star
### Start DataNode
-You could add any number of DataNode.
+You could add no less than the number of data/schema_replication_factor DataNode.
- Please set the important parameters in iotdb-datanode.properties.
+ Please set the important parameters in iotdb-datanode.properties:
| **Configuration** | **Description** |
|-------------------------------------|--------------------------------------------------|
[iotdb] 04/05: Merge branch 'master' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 90c81634e1aa0ef3785dfe575341badcdd4080a3
Merge: f50a871124 5c3f4e12fd
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu Nov 24 12:36:21 2022 +0800
Merge branch 'master' of github.com:apache/iotdb
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 203 +-
.../common/request/IndexedConsensusRequest.java | 27 +-
.../iotdb/consensus/config/MultiLeaderConfig.java | 63 +-
.../multileader/MultiLeaderConsensus.java | 4 +-
.../multileader/MultiLeaderServerImpl.java | 2 +-
.../multileader/logdispatcher/LogDispatcher.java | 153 +-
.../logdispatcher/MultiLeaderMemoryManager.java | 10 +-
.../multileader/logdispatcher/PendingBatch.java | 46 +-
.../multileader/logdispatcher/SyncStatus.java | 19 +-
.../service/MultiLeaderRPCServiceProcessor.java | 38 +-
.../multileader/logdispatcher/SyncStatusTest.java | 21 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 2 +-
docs/UserGuide/FAQ/Frequently-asked-questions.md | 2 +-
docs/UserGuide/Query-Data/Overview.md | 17 +-
docs/UserGuide/QuickStart/Files.md | 15 +-
.../Reference/ConfigNode-Config-Manual.md | 86 +-
docs/UserGuide/Reference/DataNode-Config-Manual.md | 96 +-
.../Ecosystem-Integration/Grafana-Plugin.md | 2 +-
.../zh/UserGuide/FAQ/Frequently-asked-questions.md | 2 +-
docs/zh/UserGuide/Query-Data/Overview.md | 18 +-
docs/zh/UserGuide/QuickStart/Files.md | 14 +-
.../Reference/ConfigNode-Config-Manual.md | 177 +-
.../UserGuide/Reference/DataNode-Config-Manual.md | 210 +-
grafana-plugin/pkg/plugin/plugin.go | 8 +-
.../it/IoTDBSyntaxConventionStringLiteralIT.java | 2 +-
.../operator/process/last/LastQueryOperator.java | 6 +-
.../iotdb/db/mpp/plan/analyze/TypeProvider.java | 7 -
.../db/mpp/plan/execution/QueryExecution.java | 2 +-
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 635 ++--
.../planner/distribution/DistributionPlanner.java | 2 +-
.../planner/distribution/ExchangeNodeAdder.java | 111 +
.../db/mpp/plan/planner/plan/node/PlanNode.java | 16 +-
.../plan/node/write/InsertMultiTabletsNode.java | 2 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 3 +
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 3407 --------------------
.../iotdb/db/qp/strategy/LogicalGenerator.java | 51 +-
.../mpp/execution/operator/OperatorMemoryTest.java | 4 +-
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 3 +
.../distribution/AggregationDistributionTest.java | 34 +-
.../java/org/apache/iotdb/db/qp/PlannerTest.java | 3 +
.../iotdb/db/qp/logical/LogicalPlanSmallTest.java | 3 +
.../iotdb/db/qp/other/TSPlanContextAuthorTest.java | 3 +
.../org/apache/iotdb/db/qp/sql/ASTVisitorTest.java | 66 +-
.../iotdb/db/qp/sql/IoTDBsqlVisitorTest.java | 99 -
.../iotdb/db/query/dataset/ListDataSetTest.java | 3 +
.../iotdb/db/utils/TsFileRewriteToolTest.java | 6 +-
.../src/main/thrift/mutlileader.thrift | 2 +-
47 files changed, 1110 insertions(+), 4595 deletions(-)
[iotdb] 03/05: Merge branch 'master' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f50a87112475cf91346465fe1266a33df14f2876
Merge: 65b6eee421 633cf37b1c
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Wed Nov 23 23:23:35 2022 +0800
Merge branch 'master' of github.com:apache/iotdb
.../resources/conf/iotdb-confignode.properties | 4 -
.../manager/load/LoadManagerMetrics.java | 10 +-
.../persistence/metric/NodeInfoMetrics.java | 4 +-
.../persistence/metric/PartitionInfoMetrics.java | 14 +-
.../persistence/partition/PartitionInfo.java | 1 -
.../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 2 +-
.../thrift/ConfigNodeRPCServiceMetrics.java | 2 +-
.../multileader/MultiLeaderServerMetrics.java | 4 +-
.../logdispatcher/LogDispatcherThreadMetrics.java | 4 +-
.../MultiLeaderMemoryManagerMetrics.java | 2 +-
docs/UserGuide/API/RestService.md | 60 ++---
docs/UserGuide/Write-Data/REST-API.md | 6 +-
docs/zh/UserGuide/API/RestService.md | 74 +++---
docs/zh/UserGuide/Write-Data/REST-API.md | 6 +-
grafana-plugin/go.sum | 2 +-
.../dropwizard/DropwizardMetricManager.java | 3 +-
.../dropwizard/type/DropwizardAutoGauge.java | 19 +-
.../iotdb/metrics/AbstractMetricManager.java | 134 +++++++---
.../iotdb/metrics/AbstractMetricService.java | 204 +++++++++++----
.../iotdb/metrics/DoNothingMetricService.java | 8 +-
.../apache/iotdb/metrics/config/MetricConfig.java | 255 +++++++++---------
.../metrics/config/MetricConfigDescriptor.java | 61 ++---
.../apache/iotdb/metrics/config/ReloadLevel.java | 6 +-
.../DoNothingAutoGauge.java} | 13 +-
.../iotdb/metrics/impl/DoNothingCounter.java | 2 +-
.../apache/iotdb/metrics/impl/DoNothingGauge.java | 2 +-
.../iotdb/metrics/impl/DoNothingHistogram.java | 2 +-
.../metrics/impl/DoNothingHistogramSnapshot.java | 2 +-
.../DoNothingMetric.java} | 14 +-
.../iotdb/metrics/impl/DoNothingMetricManager.java | 11 +-
.../apache/iotdb/metrics/impl/DoNothingRate.java | 2 +-
.../apache/iotdb/metrics/impl/DoNothingTimer.java | 2 +-
.../metricsets/jvm/JvmClassLoaderMetrics.java | 4 +-
.../metrics/metricsets/jvm/JvmCompileMetrics.java | 2 +-
.../iotdb/metrics/metricsets/jvm/JvmGcMetrics.java | 4 +-
.../metrics/metricsets/jvm/JvmMemoryMetrics.java | 12 +-
.../metrics/metricsets/jvm/JvmThreadMetrics.java | 8 +-
.../iotdb/metrics/reporter/CompositeReporter.java | 22 +-
.../iotdb/metrics/reporter/InternalReporter.java | 104 ++++++++
.../MemoryInternalReporter.java} | 43 ++-
.../apache/iotdb/metrics/reporter/Reporter.java | 8 +-
.../OutPutFormat.java => type/AutoGauge.java} | 15 +-
.../org/apache/iotdb/metrics/type/Counter.java | 6 +-
.../java/org/apache/iotdb/metrics/type/Gauge.java | 8 +-
.../org/apache/iotdb/metrics/type/Histogram.java | 6 +-
.../iotdb/metrics/type/HistogramSnapshot.java | 18 +-
.../org/apache/iotdb/metrics/type/IMetric.java | 10 +-
.../java/org/apache/iotdb/metrics/type/Rate.java | 14 +-
.../java/org/apache/iotdb/metrics/type/Timer.java | 10 +-
...OutPutFormat.java => InternalReporterType.java} | 8 +-
.../iotdb/metrics/utils/IoTDBMetricsUtils.java | 27 ++
.../org/apache/iotdb/metrics/utils/MetricInfo.java | 11 +-
.../iotdb/metrics/config/MetricConfigTest.java | 118 +++++----
.../micrometer/MicrometerMetricManager.java | 3 +-
.../micrometer/type/MicrometerAutoGauge.java | 19 +-
.../commons/service/metric/MetricService.java | 105 ++++----
openapi/src/main/openapi3/iotdb-rest.yaml | 17 +-
.../resources/conf/iotdb-datanode.properties | 11 +-
.../tools/tsfile/print-tsfile-visdata.bat | 62 -----
.../resources/tools/tsfile/print-tsfile-visdata.sh | 51 ----
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 18 +-
.../iotdb/db/engine/cache/ChunkCacheMetrics.java | 2 +-
.../cache/TimeSeriesMetadataCacheMetrics.java | 4 +-
.../iotdb/db/engine/flush/FlushManagerMetrics.java | 4 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 2 +-
.../storagegroup/TsFileProcessorInfoMetrics.java | 2 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 2 +-
.../rescon/SchemaStatisticsManagerMetrics.java | 2 +-
.../exchange/MPPDataExchangeServiceMetrics.java | 2 +-
...MppDataExchangeServiceThriftHandlerMetrics.java | 2 +-
.../plan/planner/plan/node/write/InsertNode.java | 7 +-
.../rest/handler/RequestValidationHandler.java | 14 +-
.../rest/handler/StatementConstructionHandler.java | 2 +-
.../pool/RawQueryReadTaskPoolManagerMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 13 +
.../service/DataNodeInternalRPCServiceMetrics.java | 2 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 12 +
.../org/apache/iotdb/db/service/RPCService.java | 8 +-
.../apache/iotdb/db/service/RPCServiceMetrics.java | 2 +-
.../db/service/metrics/DataNodeMetricsHelper.java | 14 +-
.../iotdb/db/service/metrics/FileMetrics.java | 16 +-
.../db/service/metrics/IoTDBInternalReporter.java | 172 ++++++++++++
.../iotdb/db/service/metrics/ProcessMetrics.java | 18 +-
.../iotdb/db/service/metrics/SystemMetrics.java | 16 +-
.../metrics/recorder/CacheMetricsRecorder.java | 2 +-
.../recorder/CompactionMetricsRecorder.java | 4 +-
.../InternalServiceThriftHandlerMetrics.java | 2 +-
.../handler/RPCServiceThriftHandlerMetrics.java | 2 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 20 +-
.../iotdb/db/tools/vis/TsFileExtractVisdata.java | 170 ------------
.../main/java/org/apache/iotdb/db/tools/vis/vis.m | 290 ---------------------
.../apache/iotdb/db/metric/MetricServiceTest.java | 11 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 2 -
93 files changed, 1208 insertions(+), 1296 deletions(-)
[iotdb] 01/05: Merge commit 'refs/pull/8091/head' of github.com:apache/iotdb
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 887a4d42f14e5913f3dd9044dbfd04f65867d682
Merge: 44e0419dbc 2b041fbf31
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Wed Nov 23 08:57:36 2022 +0800
Merge commit 'refs/pull/8091/head' of github.com:apache/iotdb
docs/UserGuide/Cluster/Cluster-Concept.md | 15 ++
docs/UserGuide/Cluster/Cluster-Setup.md | 210 ++++++++++---------
.../Maintenance-Tools/Maintenance-Command.md | 222 ++++++++++----------
docs/zh/UserGuide/Cluster/Cluster-Concept.md | 15 ++
docs/zh/UserGuide/Cluster/Cluster-Setup.md | 218 +++++++++++---------
.../Maintenance-Tools/Maintenance-Command.md | 223 ++++++++++-----------
.../db/mpp/common/header/ColumnHeaderConstant.java | 6 +-
7 files changed, 470 insertions(+), 439 deletions(-)
[iotdb] 05/05: perfect the migrate process
Posted by ca...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a commit to branch beyyes/fix_remove_node_problem_1123
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 822f4f1c2f9e3fd8b872dd92a77a56c76a7335f1
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Thu Nov 24 19:35:34 2022 +0800
perfect the migrate process
---
.../procedure/env/DataNodeRemoveHandler.java | 15 +-
.../iotdb/db/service/RegionMigrateService.java | 352 +++++++++------------
.../impl/DataNodeInternalRPCServiceImpl.java | 22 +-
3 files changed, 185 insertions(+), 204 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/DataNodeRemoveHandler.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/DataNodeRemoveHandler.java
index 6666d7e802..7abe82d46b 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/DataNodeRemoveHandler.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/DataNodeRemoveHandler.java
@@ -178,8 +178,17 @@ public class DataNodeRemoveHandler {
return status;
}
- List<TDataNodeLocation> currentPeerNodes = new ArrayList<>(regionReplicaNodes);
- currentPeerNodes.add(destDataNode);
+ List<TDataNodeLocation> currentPeerNodes;
+ if (TConsensusGroupType.DataRegion.equals(regionId.getType())
+ && MULTI_LEADER_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())) {
+ // parameter of createPeer for MultiLeader should be all peers
+ currentPeerNodes = new ArrayList<>(regionReplicaNodes);
+ currentPeerNodes.add(destDataNode);
+ } else {
+ // parameter of createPeer for Ratis can be empty
+ currentPeerNodes = Collections.emptyList();
+ }
+
String storageGroup = configManager.getPartitionManager().getRegionStorageGroup(regionId);
TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, storageGroup);
// TODO replace with real ttl
@@ -278,7 +287,7 @@ public class DataNodeRemoveHandler {
// Here we pick the DataNode who contains one of the RegionReplica of the specified
// ConsensusGroup except the origin one
// in order to notify the new ConsensusGroup that the origin peer should secede now
- // if the selectedDataNode equals null, we choose the destDataNode to execute the method
+ // If the selectedDataNode equals null, we choose the destDataNode to execute the method
Optional<TDataNodeLocation> selectedDataNode =
filterDataNodeWithOtherRegionReplica(regionId, originalDataNode);
rpcClientDataNode = selectedDataNode.orElse(destDataNode);
diff --git a/server/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java b/server/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java
index fc68469a3d..bc6c25f131 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/RegionMigrateService.java
@@ -52,7 +52,7 @@ import java.util.Map;
public class RegionMigrateService implements IService {
private static final Logger LOGGER = LoggerFactory.getLogger(RegionMigrateService.class);
- public static final String REMOVE_DATANODE_PROCESS = "[REMOVE_DATANODE_PROCESS]";
+ public static final String REGION_MIGRATE_PROCESS = "[REGION_MIGRATE_PROCESS]";
private static final int MAX_RETRY_NUM = 5;
@@ -67,7 +67,7 @@ public class RegionMigrateService implements IService {
}
/**
- * submit AddRegionPeerTask
+ * Submit AddRegionPeerTask
*
* @param req TMaintainPeerReq
* @return if the submit task succeed
@@ -79,9 +79,9 @@ public class RegionMigrateService implements IService {
regionMigratePool.submit(new AddRegionPeerTask(req.getRegionId(), req.getDestNode()));
} catch (Exception e) {
LOGGER.error(
- "Submit addRegionPeer task error for Region: {} on DataNode: {}.",
+ "{}, Submit AddRegionPeerTask error for Region: {}",
+ REGION_MIGRATE_PROCESS,
req.getRegionId(),
- req.getDestNode().getInternalEndPoint().getIp(),
e);
submitSucceed = false;
}
@@ -89,7 +89,7 @@ public class RegionMigrateService implements IService {
}
/**
- * submit RemoveRegionPeerTask
+ * Submit RemoveRegionPeerTask
*
* @param req TMaintainPeerReq
* @return if the submit task succeed
@@ -101,9 +101,9 @@ public class RegionMigrateService implements IService {
regionMigratePool.submit(new RemoveRegionPeerTask(req.getRegionId(), req.getDestNode()));
} catch (Exception e) {
LOGGER.error(
- "Submit removeRegionPeer task error for Region: {} on DataNode: {}.",
+ "{}, Submit RemoveRegionPeer task error for Region: {}",
+ REGION_MIGRATE_PROCESS,
req.getRegionId(),
- req.getDestNode().getInternalEndPoint().getIp(),
e);
submitSucceed = false;
}
@@ -111,10 +111,10 @@ public class RegionMigrateService implements IService {
}
/**
- * remove a region peer
+ * Submit DeleteOldRegionPeerTask
*
* @param req TMigrateRegionReq
- * @return submit task succeed?
+ * @return if the submit task succeed
*/
public synchronized boolean submitDeleteOldRegionPeerTask(TMaintainPeerReq req) {
@@ -123,9 +123,9 @@ public class RegionMigrateService implements IService {
regionMigratePool.submit(new DeleteOldRegionPeerTask(req.getRegionId(), req.getDestNode()));
} catch (Exception e) {
LOGGER.error(
- "Submit deleteOldRegionPeerTask error for Region: {} on DataNode: {}.",
+ "{}, Submit DeleteOldRegionPeerTask error for Region: {}",
+ REGION_MIGRATE_PROCESS,
req.getRegionId(),
- req.getDestNode().getInternalEndPoint().getIp(),
e);
submitSucceed = false;
}
@@ -152,17 +152,10 @@ public class RegionMigrateService implements IService {
return ServiceType.DATA_NODE_REGION_MIGRATE_SERVICE;
}
- private static class Holder {
- private static final RegionMigrateService INSTANCE = new RegionMigrateService();
-
- private Holder() {}
- }
-
private static class RegionMigratePool extends AbstractPoolManager {
private final Logger poolLogger = LoggerFactory.getLogger(RegionMigratePool.class);
private RegionMigratePool() {
- // migrate region one by one
this.pool = IoTDBThreadPoolFactory.newSingleThreadExecutor("Region-Migrate-Pool");
}
@@ -184,78 +177,25 @@ public class RegionMigrateService implements IService {
}
}
- private static void reportSucceed(TConsensusGroupId tRegionId, String migrateState) {
- TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- status.setMessage(
- String.format("Region:%s, state: %s, executed succeed", tRegionId, migrateState));
- TRegionMigrateResultReportReq req = new TRegionMigrateResultReportReq(tRegionId, status);
- try {
- reportRegionMigrateResultToConfigNode(req);
- } catch (Throwable e) {
- LOGGER.error(
- "Report region {} migrate successful result error, result: {}", tRegionId, req, e);
- }
- }
-
- private static void reportFailed(
- TConsensusGroupId tRegionId,
- TDataNodeLocation failedNode,
- TRegionMigrateFailedType failedType,
- TSStatus status) {
- TRegionMigrateResultReportReq req =
- createFailedRequest(tRegionId, failedNode, failedType, status);
- try {
- reportRegionMigrateResultToConfigNode(req);
- } catch (Throwable e) {
- LOGGER.error("Report region {} migrate failed result error, result:{}", tRegionId, req, e);
- }
- }
-
- private static TRegionMigrateResultReportReq createFailedRequest(
- TConsensusGroupId tRegionId,
- TDataNodeLocation failedNode,
- TRegionMigrateFailedType failedType,
- TSStatus status) {
- Map<TDataNodeLocation, TRegionMigrateFailedType> failedNodeAndReason = new HashMap<>();
- failedNodeAndReason.put(failedNode, failedType);
- TRegionMigrateResultReportReq req = new TRegionMigrateResultReportReq(tRegionId, status);
- req.setFailedNodeAndReason(failedNodeAndReason);
- return req;
- }
-
- private static void reportRegionMigrateResultToConfigNode(TRegionMigrateResultReportReq req)
- throws TException {
- TSStatus status;
- try (ConfigNodeClient client = new ConfigNodeClient()) {
- status = client.reportRegionMigrateResult(req);
- LOGGER.info(
- "Report region {} migrate result {} to Config node succeed, result: {}",
- req.getRegionId(),
- req,
- status);
- }
- }
-
private static class AddRegionPeerTask implements Runnable {
private static final Logger taskLogger = LoggerFactory.getLogger(AddRegionPeerTask.class);
// The RegionGroup that shall perform the add peer process
private final TConsensusGroupId tRegionId;
- // The DataNode that selected to perform the add peer process
- private final TDataNodeLocation selectedDataNode;
+ // The new DataNode to be added in the RegionGroup
+ private final TDataNodeLocation destDataNode;
- public AddRegionPeerTask(TConsensusGroupId tRegionId, TDataNodeLocation selectedDataNode) {
+ public AddRegionPeerTask(TConsensusGroupId tRegionId, TDataNodeLocation destDataNode) {
this.tRegionId = tRegionId;
- this.selectedDataNode = selectedDataNode;
+ this.destDataNode = destDataNode;
}
@Override
public void run() {
TSStatus runResult = addPeer();
if (isFailed(runResult)) {
- reportFailed(
- tRegionId, selectedDataNode, TRegionMigrateFailedType.AddPeerFailed, runResult);
+ reportFailed(tRegionId, destDataNode, TRegionMigrateFailedType.AddPeerFailed, runResult);
return;
}
@@ -263,11 +203,12 @@ public class RegionMigrateService implements IService {
}
private TSStatus addPeer() {
+ taskLogger.info(
+ "{}, Start to addPeer {} for region {}", REGION_MIGRATE_PROCESS, destDataNode, tRegionId);
ConsensusGroupId regionId = ConsensusGroupId.Factory.createFromTConsensusGroupId(tRegionId);
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
ConsensusGenericResponse resp = null;
- TEndPoint newPeerNode = getConsensusEndPoint(selectedDataNode, regionId);
- taskLogger.info("Start to addPeer {} for region {}", newPeerNode, tRegionId);
+ TEndPoint destEndpoint = getConsensusEndPoint(destDataNode, regionId);
boolean addPeerSucceed = true;
for (int i = 0; i < MAX_RETRY_NUM; i++) {
try {
@@ -276,49 +217,40 @@ public class RegionMigrateService implements IService {
}
resp =
addRegionPeer(
- regionId, new Peer(regionId, selectedDataNode.getDataNodeId(), newPeerNode));
- addPeerSucceed = true;
+ regionId, new Peer(regionId, destDataNode.getDataNodeId(), destEndpoint));
} catch (Throwable e) {
addPeerSucceed = false;
taskLogger.error(
"{}, executed addPeer {} for region {} error, retry times: {}",
- REMOVE_DATANODE_PROCESS,
- newPeerNode,
+ REGION_MIGRATE_PROCESS,
+ destEndpoint,
regionId,
i,
e);
- status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage(
- String.format(
- "Add peer for region error, peerId: %s, regionId: %s, errorMessage: %s",
- newPeerNode, regionId, e.getMessage()));
}
if (addPeerSucceed && resp != null && resp.isSuccess()) {
break;
}
}
+
if (!addPeerSucceed || resp == null || !resp.isSuccess()) {
- taskLogger.error(
- "{}, Add new peer {} for region {} failed, resp: {}",
- REMOVE_DATANODE_PROCESS,
- newPeerNode,
- regionId,
- resp);
- status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage(
+ String errorMsg =
String.format(
- "Add peer for region error, peerId: %s, regionId: %s, resp: %s",
- newPeerNode, regionId, resp));
+ "%s, AddPeer for region error after max retry times, peerId: %s, regionId: %s, resp: %s",
+ REGION_MIGRATE_PROCESS, destEndpoint, regionId, resp);
+ taskLogger.error(errorMsg);
+ status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ status.setMessage(errorMsg);
return status;
}
taskLogger.info(
"{}, Succeed to addPeer {} for region {}",
- REMOVE_DATANODE_PROCESS,
- newPeerNode,
+ REGION_MIGRATE_PROCESS,
+ destEndpoint,
regionId);
status.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- status.setMessage("add peer " + newPeerNode + " for region " + regionId + " succeed");
+ status.setMessage("addPeer " + destEndpoint + " for region " + regionId + " succeed");
return status;
}
@@ -331,22 +263,6 @@ public class RegionMigrateService implements IService {
}
return resp;
}
-
- private TEndPoint getConsensusEndPoint(
- TDataNodeLocation nodeLocation, ConsensusGroupId regionId) {
- if (regionId instanceof DataRegionId) {
- return nodeLocation.getDataRegionConsensusEndPoint();
- }
- return nodeLocation.getSchemaRegionConsensusEndPoint();
- }
-
- private boolean isSucceed(TSStatus status) {
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
- }
-
- private boolean isFailed(TSStatus status) {
- return !isSucceed(status);
- }
}
private static class RemoveRegionPeerTask implements Runnable {
@@ -354,11 +270,11 @@ public class RegionMigrateService implements IService {
private final TConsensusGroupId tRegionId;
- private final TDataNodeLocation selectedDataNode;
+ private final TDataNodeLocation destDataNode;
- public RemoveRegionPeerTask(TConsensusGroupId tRegionId, TDataNodeLocation selectedDataNode) {
+ public RemoveRegionPeerTask(TConsensusGroupId tRegionId, TDataNodeLocation destDataNode) {
this.tRegionId = tRegionId;
- this.selectedDataNode = selectedDataNode;
+ this.destDataNode = destDataNode;
}
@Override
@@ -367,29 +283,18 @@ public class RegionMigrateService implements IService {
if (isSucceed(runResult)) {
reportSucceed(tRegionId, "RemovePeer");
} else {
- reportFailed(
- tRegionId, selectedDataNode, TRegionMigrateFailedType.RemovePeerFailed, runResult);
- }
- }
-
- private ConsensusGenericResponse removeRegionPeer(ConsensusGroupId regionId, Peer oldPeer) {
- ConsensusGenericResponse resp;
- if (regionId instanceof DataRegionId) {
- resp = DataRegionConsensusImpl.getInstance().removePeer(regionId, oldPeer);
- } else {
- resp = SchemaRegionConsensusImpl.getInstance().removePeer(regionId, oldPeer);
+ reportFailed(tRegionId, destDataNode, TRegionMigrateFailedType.RemovePeerFailed, runResult);
}
- return resp;
}
private TSStatus removePeer() {
ConsensusGroupId regionId = ConsensusGroupId.Factory.createFromTConsensusGroupId(tRegionId);
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- TEndPoint oldPeerNode = getConsensusEndPoint(selectedDataNode, regionId);
+ TEndPoint destEndPoint = getConsensusEndPoint(destDataNode, regionId);
taskLogger.info(
- "{}, Start to remove peer {} for region {}",
- REMOVE_DATANODE_PROCESS,
- oldPeerNode,
+ "{}, Start to removePeer {} for region {}",
+ REGION_MIGRATE_PROCESS,
+ destEndPoint,
regionId);
ConsensusGenericResponse resp = null;
boolean removePeerSucceed = true;
@@ -400,20 +305,16 @@ public class RegionMigrateService implements IService {
}
resp =
removeRegionPeer(
- regionId, new Peer(regionId, selectedDataNode.getDataNodeId(), oldPeerNode));
- removePeerSucceed = true;
+ regionId, new Peer(regionId, destDataNode.getDataNodeId(), destEndPoint));
} catch (Throwable e) {
removePeerSucceed = false;
taskLogger.error(
- "Remove peer {} for region {} error, retry times: {}", oldPeerNode, regionId, i, e);
- status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage(
- "remove peer: "
- + oldPeerNode
- + " for region: "
- + regionId
- + " error. exception: "
- + e.getMessage());
+ "{}, executed removePeer {} for region {} error, retry times: {}",
+ REGION_MIGRATE_PROCESS,
+ destEndPoint,
+ regionId,
+ i,
+ e);
}
if (removePeerSucceed && resp != null && resp.isSuccess()) {
break;
@@ -421,41 +322,34 @@ public class RegionMigrateService implements IService {
}
if (!removePeerSucceed || resp == null || !resp.isSuccess()) {
- taskLogger.error(
- "{}, Remove old peer {} for region {} failed, resp: {}",
- REMOVE_DATANODE_PROCESS,
- oldPeerNode,
- regionId,
- resp);
+ String errorMsg =
+ String.format(
+ "%s, RemovePeer for region error after max retry times, peerId: %s, regionId: %s, resp: %s",
+ REGION_MIGRATE_PROCESS, destEndPoint, regionId, resp);
+ taskLogger.error(errorMsg);
status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage("remove old peer " + oldPeerNode + " for region " + regionId + " failed");
+ status.setMessage(errorMsg);
return status;
}
taskLogger.info(
- "{}, Succeed to remove peer {} for region {}",
- REMOVE_DATANODE_PROCESS,
- oldPeerNode,
+ "{}, Succeed to removePeer {} for region {}",
+ REGION_MIGRATE_PROCESS,
+ destEndPoint,
regionId);
status.setCode(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- status.setMessage("Remove peer " + oldPeerNode + " for region " + regionId + " succeed");
+ status.setMessage("removePeer " + destEndPoint + " for region " + regionId + " succeed");
return status;
}
- private TEndPoint getConsensusEndPoint(
- TDataNodeLocation nodeLocation, ConsensusGroupId regionId) {
+ private ConsensusGenericResponse removeRegionPeer(ConsensusGroupId regionId, Peer oldPeer) {
+ ConsensusGenericResponse resp;
if (regionId instanceof DataRegionId) {
- return nodeLocation.getDataRegionConsensusEndPoint();
+ resp = DataRegionConsensusImpl.getInstance().removePeer(regionId, oldPeer);
+ } else {
+ resp = SchemaRegionConsensusImpl.getInstance().removePeer(regionId, oldPeer);
}
- return nodeLocation.getSchemaRegionConsensusEndPoint();
- }
-
- private boolean isSucceed(TSStatus status) {
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
- }
-
- private boolean isFailed(TSStatus status) {
- return !isSucceed(status);
+ return resp;
}
}
@@ -464,32 +358,43 @@ public class RegionMigrateService implements IService {
private final TConsensusGroupId tRegionId;
- private final TDataNodeLocation fromNode;
+ private final TDataNodeLocation originalDataNode;
- public DeleteOldRegionPeerTask(TConsensusGroupId tRegionId, TDataNodeLocation fromNode) {
+ public DeleteOldRegionPeerTask(
+ TConsensusGroupId tRegionId, TDataNodeLocation originalDataNode) {
this.tRegionId = tRegionId;
- this.fromNode = fromNode;
+ this.originalDataNode = originalDataNode;
}
@Override
public void run() {
- TSStatus runResult = deleteOldRegionPeer();
+ // deletePeer: remove the peer from the consensus group
+ TSStatus runResult = deletePeer();
if (isFailed(runResult)) {
reportFailed(
- tRegionId, fromNode, TRegionMigrateFailedType.RemoveConsensusGroupFailed, runResult);
+ tRegionId,
+ originalDataNode,
+ TRegionMigrateFailedType.RemoveConsensusGroupFailed,
+ runResult);
}
+ // deleteRegion: delete region data
runResult = deleteRegion();
if (isFailed(runResult)) {
- reportFailed(tRegionId, fromNode, TRegionMigrateFailedType.DeleteRegionFailed, runResult);
+ reportFailed(
+ tRegionId, originalDataNode, TRegionMigrateFailedType.DeleteRegionFailed, runResult);
}
reportSucceed(tRegionId, "DeletePeer");
}
- private TSStatus deleteOldRegionPeer() {
+ private TSStatus deletePeer() {
+ taskLogger.info(
+ "{}, Start to deletePeer {} for region {}",
+ REGION_MIGRATE_PROCESS,
+ originalDataNode,
+ tRegionId);
ConsensusGroupId regionId = ConsensusGroupId.Factory.createFromTConsensusGroupId(tRegionId);
- taskLogger.info("Start to deleteOldRegionPeer: {}", regionId);
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
ConsensusGenericResponse resp;
try {
@@ -499,30 +404,31 @@ public class RegionMigrateService implements IService {
resp = SchemaRegionConsensusImpl.getInstance().deletePeer(regionId);
}
} catch (Throwable e) {
- taskLogger.error("DeleteOldRegionPeer error, regionId: {}", regionId, e);
+ taskLogger.error("{}, deletePeer error, regionId: {}", regionId, e);
status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
status.setMessage(
- "deleteOldRegionPeer for region: " + regionId + " error. exception: " + e.getMessage());
+ "deletePeer for region: " + regionId + " error. exception: " + e.getMessage());
return status;
}
if (!resp.isSuccess()) {
- taskLogger.error("DeleteOldRegionPeer error, regionId: {}", regionId, resp.getException());
- status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage(
+ String errorMsg =
String.format(
- "deleteOldRegionPeer error, regionId: %s, errorMessage: %s",
- regionId, resp.getException().getMessage()));
+ "deletePeer error, regionId: %s, errorMessage: %s",
+ regionId, resp.getException().getMessage());
+ taskLogger.error(errorMsg);
+ status.setCode(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ status.setMessage(errorMsg);
return status;
}
- taskLogger.info("Succeed to remove region {} consensus group", regionId);
- status.setMessage("remove region consensus group " + regionId + "succeed");
+ taskLogger.info("Succeed to deletePeer {} from consensus group", regionId);
+ status.setMessage("deletePeer from consensus group " + regionId + "succeed");
return status;
}
private TSStatus deleteRegion() {
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
ConsensusGroupId regionId = ConsensusGroupId.Factory.createFromTConsensusGroupId(tRegionId);
- taskLogger.info("Start to delete region {}", regionId);
+ taskLogger.info("{}, Start to deleteRegion {}", REGION_MIGRATE_PROCESS, regionId);
try {
if (regionId instanceof DataRegionId) {
StorageEngineV2.getInstance().deleteDataRegion((DataRegionId) regionId);
@@ -530,22 +436,78 @@ public class RegionMigrateService implements IService {
SchemaEngine.getInstance().deleteSchemaRegion((SchemaRegionId) regionId);
}
} catch (Throwable e) {
- taskLogger.error("Delete the region {} failed", regionId, e);
+ taskLogger.error("deleteRegion {} failed", regionId, e);
status.setCode(TSStatusCode.DELETE_REGION_ERROR.getStatusCode());
- status.setMessage("Delete region " + regionId + "failed, " + e.getMessage());
+ status.setMessage("deleteRegion " + regionId + "failed, " + e.getMessage());
return status;
}
- status.setMessage("delete region " + regionId + " succeed");
- taskLogger.info("Finished to delete region {}", regionId);
+ status.setMessage("deleteRegion " + regionId + " succeed");
+ taskLogger.info("Succeed to deleteRegion {}", regionId);
return status;
}
+ }
+
+ private static class Holder {
+ private static final RegionMigrateService INSTANCE = new RegionMigrateService();
+
+ private Holder() {}
+ }
+
+ private static void reportSucceed(TConsensusGroupId tRegionId, String migrateState) {
+ TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ status.setMessage(
+ String.format("Region: %s, state: %s, executed succeed", tRegionId, migrateState));
+ TRegionMigrateResultReportReq req = new TRegionMigrateResultReportReq(tRegionId, status);
+ try {
+ reportRegionMigrateResultToConfigNode(req);
+ } catch (Throwable e) {
+ LOGGER.error(
+ "Report region {} migrate result error in reportSucceed, result: {}", tRegionId, req, e);
+ }
+ }
+
+ private static void reportFailed(
+ TConsensusGroupId tRegionId,
+ TDataNodeLocation failedNode,
+ TRegionMigrateFailedType failedType,
+ TSStatus status) {
+ Map<TDataNodeLocation, TRegionMigrateFailedType> failedNodeAndReason = new HashMap<>();
+ failedNodeAndReason.put(failedNode, failedType);
+ TRegionMigrateResultReportReq req = new TRegionMigrateResultReportReq(tRegionId, status);
+ req.setFailedNodeAndReason(failedNodeAndReason);
+ try {
+ reportRegionMigrateResultToConfigNode(req);
+ } catch (Throwable e) {
+ LOGGER.error("Report region {} migrate error in reportFailed, result:{}", tRegionId, req, e);
+ }
+ }
- private boolean isSucceed(TSStatus status) {
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ private static void reportRegionMigrateResultToConfigNode(TRegionMigrateResultReportReq req)
+ throws TException {
+ TSStatus status;
+ try (ConfigNodeClient client = new ConfigNodeClient()) {
+ status = client.reportRegionMigrateResult(req);
+ LOGGER.info(
+ "Report region {} migrate result {} to Config node succeed, result: {}",
+ req.getRegionId(),
+ req,
+ status);
}
+ }
+
+ private static boolean isSucceed(TSStatus status) {
+ return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ }
+
+ private static boolean isFailed(TSStatus status) {
+ return !isSucceed(status);
+ }
- private boolean isFailed(TSStatus status) {
- return !isSucceed(status);
+ private static TEndPoint getConsensusEndPoint(
+ TDataNodeLocation nodeLocation, ConsensusGroupId regionId) {
+ if (regionId instanceof DataRegionId) {
+ return nodeLocation.getDataRegionConsensusEndPoint();
}
+ return nodeLocation.getSchemaRegionConsensusEndPoint();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 559684bd34..8149b5534e 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -201,6 +201,7 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.stream.Collectors;
import static org.apache.iotdb.commons.conf.IoTDBConstant.MULTI_LEVEL_PATH_WILDCARD;
+import static org.apache.iotdb.db.service.RegionMigrateService.REGION_MIGRATE_PROCESS;
import static org.apache.iotdb.db.service.basic.ServiceProvider.QUERY_FREQUENCY_RECORDER;
import static org.apache.iotdb.db.utils.ErrorHandlingUtils.onQueryException;
@@ -1261,7 +1262,7 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
if (submitSucceed) {
LOGGER.info(
- "Successfully submit addRegionPeer task for region: {} on DataNode: {}",
+ "Successfully submit addRegionPeer task for region: {}, target DataNode: {}",
regionId,
selectedDataNodeIP);
return status;
@@ -1279,7 +1280,7 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
if (submitSucceed) {
LOGGER.info(
- "Successfully submit removeRegionPeer task for region: {} on DataNode: {}",
+ "Successfully submit removeRegionPeer task for region: {}, DataNode to be removed: {}",
regionId,
selectedDataNodeIP);
return status;
@@ -1297,7 +1298,7 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
if (submitSucceed) {
LOGGER.info(
- "Successfully submit deleteOldRegionPeer task for region: {} on DataNode: {}",
+ "Successfully submit deleteOldRegionPeer task for region: {}, DataNode to be removed: {}",
regionId,
selectedDataNodeIP);
return status;
@@ -1455,7 +1456,11 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
}
private TSStatus createNewRegionPeer(ConsensusGroupId regionId, List<Peer> peers) {
- LOGGER.info("Start to createNewRegionPeer {} to region {}", peers, regionId);
+ LOGGER.info(
+ "{}, Start to createNewRegionPeer {} to region {}",
+ REGION_MIGRATE_PROCESS,
+ peers,
+ regionId);
TSStatus status = new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
ConsensusGenericResponse resp;
if (regionId instanceof DataRegionId) {
@@ -1465,7 +1470,8 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
}
if (!resp.isSuccess()) {
LOGGER.error(
- "CreateNewRegionPeer error, peers: {}, regionId: {}, errorMessage",
+ "{}, CreateNewRegionPeer error, peers: {}, regionId: {}, errorMessage",
+ REGION_MIGRATE_PROCESS,
peers,
regionId,
resp.getException());
@@ -1473,7 +1479,11 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
status.setMessage(resp.getException().getMessage());
return status;
}
- LOGGER.info("Succeed to createNewRegionPeer {} for region {}", peers, regionId);
+ LOGGER.info(
+ "{}, Succeed to createNewRegionPeer {} for region {}",
+ REGION_MIGRATE_PROCESS,
+ peers,
+ regionId);
status.setMessage("createNewRegionPeer succeed, regionId: " + regionId);
return status;
}