You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/10/27 08:47:22 UTC
[iotdb] branch xingtanzjr/migration_add_wait_sync updated (133f22ef27 -> 74c5ee4b7d)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a change to branch xingtanzjr/migration_add_wait_sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 133f22ef27 add logic to wail log sync to be completed when some peer is going to be removed
new c5098b4a70 tmp save
add eb051006ba Lowering Partition related logs' level (#7714)
add 6f27466c9a Use 1c1d docker to run go client IT (#7718)
add 82253fa242 [IOTDB-4734] Modify create function grammar and process (#7719)
add 8c2b803ce4 Rerun a new query execution thread while previous one is interrupted (#7722)
add 8a68f3a8ce [IOTDB-4751] Fix GROUP BY TAGS being without any error message (#7720)
add 531f8b3533 [IOTDB-4732] Optimize client borrow logic in ClusterIT environment (#7713)
add 5297117206 [IOTDB-4736] Refactor process of show functions (#7727)
add c69748d613 [IOTDB-4742] [Consensus] add new api: addNewNodeToExistedGroup (#7712)
add 55c1ea0edf [IOTDB-4721] Modify Create UDF process on ConfigNode (#7730)
add b268472003 [IOTDB-4756] Register UDF when restart/register DataNode
add ac8283a814 [IOTDB-4616] Support serialization and deserialization for confignode standalone mode (#7731)
add 3dda1ab202 [IOTDB-4757][IOTDB-4754]Fix concurrent register schema bug and show timeseries null element (#7728)
add 9f8bcc7692 [IOTDB-4651] Remove Predefined Metric Sets and Add MultiLeader metrics. (#7602)
add 3130ce9a5f [IOTDB-4660] Implement IntoOperator & ITs (#7665)
add da959a6f24 [IOTDB-4713] Log TException in ConfigNodeClient (#7729)
add 56ef925047 [IOTDB-4667] update the detailed usage method to the cpp-client readme doc (#7640)
add c26f6559e8 [IOTDB-4722][IOTDB-4723]modify Show & Drop UDF process on ConfigNode
add baed39decc [IOTDB-4693] Support broken tsfile rewrite (#7677)
add 89e5957683 Change AsyncDataNodeInternalServiceClient level from info to error and make it simple (#7747)
add dff3d3727e [IOTDB-4679] Unable to connect to iotdb in private-network using port-mapping (#7721)
add 0767935311 [IOTDB-4758]Delete snapshot after region migration (#7739)
new 31c750172a Merge branch 'master' into xingtanzjr/migration_add_wait_sync
new 74c5ee4b7d complete wait logic when migrate dataregion for ML
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/client-go.yml | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +-
client-cpp/README.md | 41 +-
client-go | 2 +-
client-py/tests/test_dataframe.py | 4 +-
.../client/async/AsyncDataNodeClientPool.java | 8 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +
.../confignode/conf/ConfigNodeDescriptor.java | 6 +
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 3 +-
.../request/read/GetFunctionTablePlan.java | 25 +-
.../request/write/function/CreateFunctionPlan.java | 52 +-
.../consensus/response/FunctionTableResp.java | 70 +++
.../iotdb/confignode/manager/ConfigManager.java | 27 +-
.../iotdb/confignode/manager/ConsensusManager.java | 180 +++---
.../apache/iotdb/confignode/manager/IManager.java | 6 +-
.../iotdb/confignode/manager/UDFManager.java | 122 ++--
.../iotdb/confignode/manager/load/LoadManager.java | 2 +-
.../manager/load/LoadManagerMetrics.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 2 +-
.../iotdb/confignode/persistence/UDFInfo.java | 119 ++--
.../persistence/executor/ConfigPlanExecutor.java | 4 +-
.../persistence/metric/NodeInfoMetrics.java | 4 +-
.../persistence/metric/PartitionInfoMetrics.java | 4 +-
.../persistence/partition/PartitionInfo.java | 6 +-
.../iotdb/confignode/service/ConfigNode.java | 12 +-
.../service/thrift/ConfigNodeRPCService.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandler.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 17 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 9 +
.../resources/confignode1conf/iotdb-metric.yml | 5 -
.../resources/confignode2conf/iotdb-metric.yml | 5 -
.../resources/confignode3conf/iotdb-metric.yml | 5 -
.../org/apache/iotdb/consensus/IConsensus.java | 24 +
.../multileader/MultiLeaderConsensus.java | 12 +
.../multileader/MultiLeaderServerImpl.java | 101 ++++
.../multileader/MultiLeaderServerMetrics.java | 88 +++
.../multileader/client/DispatchLogHandler.java | 15 +
.../multileader/logdispatcher/LogDispatcher.java | 23 +
.../logdispatcher/LogDispatcherThreadMetrics.java | 95 ++++
.../logdispatcher/MultiLeaderMemoryManager.java | 10 +-
.../MultiLeaderMemoryManagerMetrics.java | 41 +-
.../service/MultiLeaderRPCServiceProcessor.java | 40 ++
.../iotdb/consensus/ratis/RatisConsensus.java | 60 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 14 +
docs/UserGuide/API/Interface-Comparison.md | 44 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 6 -
docs/zh/UserGuide/API/Interface-Comparison.md | 44 +-
.../UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 7 -
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 23 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 8 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +-
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 8 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 3 +
.../it/IoTDBSyntaxConventionStringLiteralIT.java | 5 +
.../db/it/aggregation/IoTDBTagAggregationIT.java | 16 +
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 4 +-
.../iotdb/db/it/env/StandaloneEnvConfig.java | 13 +
.../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 59 ++
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 528 ++++++++++++++++++
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 3 +
.../db/it/udf/IoTDBUDFOtherWindowQueryIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java | 2 +
.../db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java | 3 +
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 2 +
.../iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java | 2 +
.../org/apache/iotdb/db/it/utils/TestUtils.java | 10 +
.../src/test/resources/logback-test.xml | 2 +-
.../db/integration/IoTDBRewriteTsFileToolIT.java | 523 ++++++++++++++++-
.../apache/iotdb/session/pool/SessionPoolTest.java | 4 +-
integration/src/test/resources/logback-test.xml | 2 +-
metrics/ReadMe.md | 9 +-
.../resources/conf/iotdb-confignode-metric.yml | 5 -
.../resources/conf/iotdb-datanode-metric.yml | 5 -
.../iotdb/metrics/AbstractMetricService.java | 19 +-
.../iotdb/metrics/DoNothingMetricService.java | 6 -
.../apache/iotdb/metrics/config/MetricConfig.java | 15 -
.../metrics/config/MetricConfigDescriptor.java | 1 -
.../jvm/JvmClassLoaderMetrics.java | 2 +-
.../{predefined => }/jvm/JvmCompileMetrics.java | 2 +-
.../{predefined => }/jvm/JvmGcMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMemoryMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMetrics.java | 2 +-
.../{predefined => }/jvm/JvmThreadMetrics.java | 2 +-
.../metricsets/{predefined => }/jvm/JvmUtils.java | 2 +-
.../{predefined => }/logback/LogbackMetrics.java | 2 +-
.../logback/MetricsTurboFilter.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 1 -
.../interface/src/test/resources/iotdb-metric.yml | 8 -
node-commons/pom.xml | 21 +
.../async/AsyncDataNodeInternalServiceClient.java | 2 +-
.../commons/service/metric}/MetricService.java | 35 +-
.../service/metric}/MetricServiceMBean.java | 2 +-
.../commons/service/metric}/enums/Metric.java | 9 +-
.../commons/service/metric}/enums/Operation.java | 2 +-
.../iotdb/commons/service/metric}/enums/Tag.java | 3 +-
.../apache/iotdb/commons/udf/UDFInformation.java | 83 ++-
.../org/apache/iotdb/commons/udf/UDFTable.java | 57 +-
.../commons/udf/service/UDFManagementService.java | 162 ++++--
.../java/org/apache/iotdb/RewriteTsFileTool.java | 620 +++++++++++++++------
server/pom.xml | 10 -
.../resources/conf/iotdb-datanode.properties | 1 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 330 +++++++++++
.../iotdb/db/client/DataNodeInternalClient.java | 129 +++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../iotdb/db/engine/TsFileMetricManager.java | 7 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../iotdb/db/engine/cache/ChunkCacheMetrics.java | 4 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
.../cache/TimeSeriesMetadataCacheMetrics.java | 4 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 2 +-
.../iotdb/db/engine/flush/FlushManagerMetrics.java | 4 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 6 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 2 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 2 +-
.../storagegroup/TsFileProcessorInfoMetrics.java | 6 +-
.../iotdb/db/exception/IntoProcessException.java | 16 +-
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 4 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 247 ++++----
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 128 +++--
.../db/metadata/rescon/SchemaResourceManager.java | 2 +-
.../rescon/SchemaStatisticsManagerMetrics.java | 4 +-
.../apache/iotdb/db/mpp/common/SessionInfo.java | 19 +
.../execution/exchange/MPPDataExchangeService.java | 2 +-
.../exchange/MPPDataExchangeServiceMetrics.java | 4 +-
.../MPPDataExchangeServiceThriftHandler.java | 2 +-
...MppDataExchangeServiceThriftHandlerMetrics.java | 6 +-
.../fragment/FragmentInstanceContext.java | 29 +-
.../fragment/FragmentInstanceManager.java | 6 +-
.../db/mpp/execution/operator/OperatorContext.java | 5 +
.../operator/process/AbstractIntoOperator.java | 384 +++++++++++++
.../operator/process/DeviceViewIntoOperator.java | 140 +++++
.../execution/operator/process/IntoOperator.java | 104 ++++
.../process/RawDataAggregationOperator.java | 2 +-
.../process/SingleInputAggregationOperator.java | 2 -
.../process/SlidingWindowAggregationOperator.java | 2 +-
.../schema/TimeSeriesSchemaScanOperator.java | 9 +-
.../execution/schedule/AbstractDriverThread.java | 60 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 38 +-
.../mpp/execution/schedule/DriverTaskThread.java | 5 +-
.../schedule/DriverTaskTimeoutSentinelThread.java | 5 +-
.../execution/schedule/ThreadProducer.java} | 19 +-
.../apache/iotdb/db/mpp/plan/TestRPCClient.java | 21 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 8 +-
.../config/executor/ClusterConfigTaskExecutor.java | 113 +++-
.../config/executor/IConfigTaskExecutor.java | 8 +-
.../executor/StandaloneConfigTaskExecutor.java | 21 +-
.../config/metadata/CreateFunctionTask.java | 15 +-
.../config/metadata/ShowFunctionsTask.java | 155 ++----
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 59 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 40 ++
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 7 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 123 ++++
.../planner/distribution/ExchangeNodeAdder.java | 6 +-
.../SimpleFragmentParallelPlanner.java | 1 +
.../plan/planner/distribution/SourceRewriter.java | 11 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../db/mpp/plan/planner/plan/FragmentInstance.java | 22 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 10 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 10 +
.../metedata/read/SchemaQueryOrderByHeatNode.java | 4 +-
.../planner/plan/node/process/AggregationNode.java | 23 +-
.../planner/plan/node/process/DeviceMergeNode.java | 32 +-
.../{TimeJoinNode.java => DeviceViewIntoNode.java} | 95 ++--
.../planner/plan/node/process/DeviceViewNode.java | 19 +-
.../planner/plan/node/process/ExchangeNode.java | 41 +-
.../plan/planner/plan/node/process/FillNode.java | 46 +-
.../plan/node/process/GroupByLevelNode.java | 29 +-
.../planner/plan/node/process/GroupByTagNode.java | 17 +-
.../node/process/{LimitNode.java => IntoNode.java} | 94 ++--
.../plan/planner/plan/node/process/LimitNode.java | 37 +-
...tiChildNode.java => MultiChildProcessNode.java} | 23 +-
.../plan/planner/plan/node/process/OffsetNode.java | 36 +-
.../planner/plan/node/process/ProjectNode.java | 40 +-
...iChildNode.java => SingleChildProcessNode.java} | 50 +-
.../node/process/SlidingWindowAggregationNode.java | 36 +-
.../plan/planner/plan/node/process/SortNode.java | 29 +-
.../planner/plan/node/process/TimeJoinNode.java | 17 +-
.../planner/plan/node/process/TransformNode.java | 29 +-
.../node/process/last/LastQueryCollectNode.java | 4 +-
.../plan/node/process/last/LastQueryMergeNode.java | 4 +-
.../plan/node/process/last/LastQueryNode.java | 4 +-
.../parameter/DeviceViewIntoPathDescriptor.java | 11 +-
.../planner/plan/parameter/IntoPathDescriptor.java | 10 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 6 +-
.../plan/statement/component/IntoComponent.java | 8 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 19 +-
.../metadata/CreateFunctionStatement.java | 21 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 22 +-
.../db/query/pool/RawQueryReadTaskPoolManager.java | 2 +-
.../pool/RawQueryReadTaskPoolManagerMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 128 ++++-
.../db/service/DataNodeInternalRPCService.java | 2 +-
.../service/DataNodeInternalRPCServiceMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 5 +-
.../org/apache/iotdb/db/service/RPCService.java | 2 +-
.../apache/iotdb/db/service/RPCServiceMetrics.java | 4 +-
.../db/service/ResourcesInformationHolder.java | 12 +
...erviceMBean.java => DataNodeMetricsHelper.java} | 25 +-
.../metrics/{predefined => }/FileMetrics.java | 6 +-
.../metrics/{predefined => }/ProcessMetrics.java | 6 +-
.../metrics/{predefined => }/SystemMetrics.java | 6 +-
.../metrics/recorder/CacheMetricsRecorder.java | 6 +-
.../recorder/CompactionMetricsRecorder.java | 6 +-
.../db/service/thrift/ProcessorWithMetrics.java | 6 +-
.../handler/InternalServiceThriftHandler.java | 2 +-
.../InternalServiceThriftHandlerMetrics.java | 4 +-
.../thrift/handler/RPCServiceThriftHandler.java | 2 +-
.../handler/RPCServiceThriftHandlerMetrics.java | 4 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 32 +-
.../db/service/thrift/impl/TSServiceImpl.java | 4 +-
.../trigger/service/TriggerManagementService.java | 18 +-
.../DriverTaskTimeoutSentinelThreadTest.java | 23 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeFailTest.java | 87 ++-
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 8 +-
.../mpp/plan/plan/FragmentInstanceSerdeTest.java | 10 +-
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 40 ++
.../plan/plan/node/process/IntoNodeSerdeTest.java | 83 +++
.../plan/scheduler/StandaloneSchedulerTest.java | 18 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 8 +-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 2 +
.../test/resources/datanode1conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode2conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode3conf/iotdb-metric.yml | 7 +-
server/src/test/resources/logback-test.xml | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 5 +
.../java/org/apache/iotdb/session/ISession.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 258 ++++++---
.../org/apache/iotdb/session/SessionConfig.java | 2 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 48 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 4 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 2 +-
.../java/org/apache/iotdb/db/sql/ClusterIT.java | 2 +-
.../src/main/thrift/confignode.thrift | 47 +-
.../src/main/thrift/mutlileader.thrift | 10 +
thrift/src/main/thrift/datanode.thrift | 16 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 72 ++-
.../java/org/apache/iotdb/tsfile/utils/BitMap.java | 6 +
251 files changed, 6046 insertions(+), 1975 deletions(-)
copy server/src/main/java/org/apache/iotdb/db/service/metrics/enums/Tag.java => confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/read/GetFunctionTablePlan.java (54%)
create mode 100644 confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/FunctionTableResp.java
create mode 100644 consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerMetrics.java
create mode 100644 consensus/src/main/java/org/apache/iotdb/consensus/multileader/logdispatcher/LogDispatcherThreadMetrics.java
copy server/src/main/java/org/apache/iotdb/db/engine/cache/ChunkCacheMetrics.java => consensus/src/main/java/org/apache/iotdb/consensus/multileader/logdispatcher/MultiLeaderMemoryManagerMetrics.java (58%)
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectInto2IT.java
create mode 100644 integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmClassLoaderMetrics.java (97%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmCompileMetrics.java (97%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmGcMetrics.java (99%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmMemoryMetrics.java (98%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmMetrics.java (96%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmThreadMetrics.java (98%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/jvm/JvmUtils.java (96%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/logback/LogbackMetrics.java (98%)
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/{predefined => }/logback/MetricsTurboFilter.java (98%)
rename {server/src/main/java/org/apache/iotdb/db/service/metrics => node-commons/src/main/java/org/apache/iotdb/commons/service/metric}/MetricService.java (76%)
copy {server/src/main/java/org/apache/iotdb/db/service/metrics => node-commons/src/main/java/org/apache/iotdb/commons/service/metric}/MetricServiceMBean.java (95%)
rename {server/src/main/java/org/apache/iotdb/db/service/metrics => node-commons/src/main/java/org/apache/iotdb/commons/service/metric}/enums/Metric.java (92%)
rename {server/src/main/java/org/apache/iotdb/db/service/metrics => node-commons/src/main/java/org/apache/iotdb/commons/service/metric}/enums/Operation.java (96%)
copy {server/src/main/java/org/apache/iotdb/db/service/metrics => node-commons/src/main/java/org/apache/iotdb/commons/service/metric}/enums/Tag.java (93%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/client/DataNodeInternalClient.java
rename metrics/interface/src/main/java/org/apache/iotdb/metrics/metricsets/predefined/PredefinedMetric.java => server/src/main/java/org/apache/iotdb/db/exception/IntoProcessException.java (75%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/AbstractIntoOperator.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/DeviceViewIntoOperator.java
create mode 100644 server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/IntoOperator.java
rename server/src/main/java/org/apache/iotdb/db/{service/metrics/enums/Tag.java => mpp/execution/schedule/ThreadProducer.java} (66%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/{TimeJoinNode.java => DeviceViewIntoNode.java} (50%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/{LimitNode.java => IntoNode.java} (55%)
copy server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/{MultiChildNode.java => MultiChildProcessNode.java} (77%)
rename server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/process/{MultiChildNode.java => SingleChildProcessNode.java} (59%)
rename server/src/main/java/org/apache/iotdb/db/service/metrics/{MetricServiceMBean.java => DataNodeMetricsHelper.java} (50%)
rename server/src/main/java/org/apache/iotdb/db/service/metrics/{predefined => }/FileMetrics.java (97%)
rename server/src/main/java/org/apache/iotdb/db/service/metrics/{predefined => }/ProcessMetrics.java (97%)
rename server/src/main/java/org/apache/iotdb/db/service/metrics/{predefined => }/SystemMetrics.java (98%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/mpp/plan/plan/node/process/IntoNodeSerdeTest.java
[iotdb] 02/03: Merge branch 'master' into xingtanzjr/migration_add_wait_sync
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/migration_add_wait_sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 31c750172ace3f11eee0f50055bfbae0979a568b
Merge: c5098b4a70 0767935311
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Oct 27 14:13:45 2022 +0800
Merge branch 'master' into xingtanzjr/migration_add_wait_sync
.github/workflows/client-go.yml | 2 +-
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 4 +-
client-cpp/README.md | 41 +-
client-go | 2 +-
client-py/tests/test_dataframe.py | 4 +-
.../client/async/AsyncDataNodeClientPool.java | 8 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 10 +
.../confignode/conf/ConfigNodeDescriptor.java | 6 +
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 3 +-
.../request/read/GetFunctionTablePlan.java | 25 +-
.../request/write/function/CreateFunctionPlan.java | 52 +-
.../consensus/response/FunctionTableResp.java | 70 +++
.../iotdb/confignode/manager/ConfigManager.java | 27 +-
.../iotdb/confignode/manager/ConsensusManager.java | 180 +++---
.../apache/iotdb/confignode/manager/IManager.java | 6 +-
.../iotdb/confignode/manager/UDFManager.java | 122 ++--
.../iotdb/confignode/manager/load/LoadManager.java | 2 +-
.../manager/load/LoadManagerMetrics.java | 4 +-
.../iotdb/confignode/manager/node/NodeManager.java | 2 +-
.../manager/partition/PartitionManager.java | 2 +-
.../iotdb/confignode/persistence/UDFInfo.java | 119 ++--
.../persistence/executor/ConfigPlanExecutor.java | 4 +-
.../persistence/metric/NodeInfoMetrics.java | 4 +-
.../persistence/metric/PartitionInfoMetrics.java | 4 +-
.../persistence/partition/PartitionInfo.java | 6 +-
.../iotdb/confignode/service/ConfigNode.java | 12 +-
.../service/thrift/ConfigNodeRPCService.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandler.java | 2 +-
.../thrift/ConfigNodeRPCServiceHandlerMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceMetrics.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 17 +-
.../request/ConfigPhysicalPlanSerDeTest.java | 9 +
.../resources/confignode1conf/iotdb-metric.yml | 5 -
.../resources/confignode2conf/iotdb-metric.yml | 5 -
.../resources/confignode3conf/iotdb-metric.yml | 5 -
.../org/apache/iotdb/consensus/IConsensus.java | 24 +
.../multileader/MultiLeaderConsensus.java | 12 +
.../multileader/MultiLeaderServerImpl.java | 96 ++++
.../multileader/MultiLeaderServerMetrics.java | 88 +++
.../multileader/client/DispatchLogHandler.java | 15 +
.../multileader/logdispatcher/LogDispatcher.java | 23 +
.../logdispatcher/LogDispatcherThreadMetrics.java | 95 ++++
.../logdispatcher/MultiLeaderMemoryManager.java | 10 +-
.../MultiLeaderMemoryManagerMetrics.java | 41 +-
.../service/MultiLeaderRPCServiceProcessor.java | 31 ++
.../iotdb/consensus/ratis/RatisConsensus.java | 60 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 14 +
docs/UserGuide/API/Interface-Comparison.md | 44 +-
docs/UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 6 -
docs/zh/UserGuide/API/Interface-Comparison.md | 44 +-
.../UserGuide/API/Programming-Java-Native-API.md | 2 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 7 -
.../java/org/apache/iotdb/it/env/AbstractEnv.java | 23 +-
.../java/org/apache/iotdb/it/env/MppConfig.java | 8 +
.../org/apache/iotdb/itbase/env/BaseConfig.java | 8 +
.../java/org/apache/iotdb/itbase/env/BaseEnv.java | 8 +-
.../org/apache/iotdb/db/it/IoTDBNestedQueryIT.java | 8 +-
.../db/it/IoTDBSyntaxConventionIdentifierIT.java | 3 +
.../it/IoTDBSyntaxConventionStringLiteralIT.java | 5 +
.../db/it/aggregation/IoTDBTagAggregationIT.java | 16 +
.../org/apache/iotdb/db/it/env/StandaloneEnv.java | 4 +-
.../iotdb/db/it/env/StandaloneEnvConfig.java | 13 +
.../iotdb/db/it/selectinto/IoTDBSelectInto2IT.java | 59 ++
.../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 528 ++++++++++++++++++
.../iotdb/db/it/udf/IoTDBUDFManagementIT.java | 3 +
.../db/it/udf/IoTDBUDFOtherWindowQueryIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDFWindowQueryIT.java | 2 +
.../db/it/udf/IoTDBUDTFAlignByTimeQueryIT.java | 3 +
.../db/it/udf/IoTDBUDTFBuiltinFunctionIT.java | 3 +
.../iotdb/db/it/udf/IoTDBUDTFHybridQueryIT.java | 2 +
.../iotdb/db/it/udf/IoTDBUDTFNonAlignQueryIT.java | 2 +
.../org/apache/iotdb/db/it/utils/TestUtils.java | 10 +
.../src/test/resources/logback-test.xml | 2 +-
.../db/integration/IoTDBRewriteTsFileToolIT.java | 523 ++++++++++++++++-
.../apache/iotdb/session/pool/SessionPoolTest.java | 4 +-
integration/src/test/resources/logback-test.xml | 2 +-
metrics/ReadMe.md | 9 +-
.../resources/conf/iotdb-confignode-metric.yml | 5 -
.../resources/conf/iotdb-datanode-metric.yml | 5 -
.../iotdb/metrics/AbstractMetricService.java | 19 +-
.../iotdb/metrics/DoNothingMetricService.java | 6 -
.../apache/iotdb/metrics/config/MetricConfig.java | 15 -
.../metrics/config/MetricConfigDescriptor.java | 1 -
.../jvm/JvmClassLoaderMetrics.java | 2 +-
.../{predefined => }/jvm/JvmCompileMetrics.java | 2 +-
.../{predefined => }/jvm/JvmGcMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMemoryMetrics.java | 2 +-
.../{predefined => }/jvm/JvmMetrics.java | 2 +-
.../{predefined => }/jvm/JvmThreadMetrics.java | 2 +-
.../metricsets/{predefined => }/jvm/JvmUtils.java | 2 +-
.../{predefined => }/logback/LogbackMetrics.java | 2 +-
.../logback/MetricsTurboFilter.java | 2 +-
.../iotdb/metrics/config/MetricConfigTest.java | 1 -
.../interface/src/test/resources/iotdb-metric.yml | 8 -
node-commons/pom.xml | 21 +
.../async/AsyncDataNodeInternalServiceClient.java | 2 +-
.../commons/service/metric}/MetricService.java | 35 +-
.../service/metric}/MetricServiceMBean.java | 2 +-
.../commons/service/metric}/enums/Metric.java | 9 +-
.../commons/service/metric}/enums/Operation.java | 2 +-
.../iotdb/commons/service/metric}/enums/Tag.java | 3 +-
.../apache/iotdb/commons/udf/UDFInformation.java | 83 ++-
.../org/apache/iotdb/commons/udf/UDFTable.java | 57 +-
.../commons/udf/service/UDFManagementService.java | 162 ++++--
.../java/org/apache/iotdb/RewriteTsFileTool.java | 620 +++++++++++++++------
server/pom.xml | 10 -
.../resources/conf/iotdb-datanode.properties | 1 +
.../apache/iotdb/db/client/ConfigNodeClient.java | 330 +++++++++++
.../iotdb/db/client/DataNodeInternalClient.java | 129 +++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 5 -
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../iotdb/db/engine/TsFileMetricManager.java | 7 +-
.../apache/iotdb/db/engine/cache/ChunkCache.java | 2 +-
.../iotdb/db/engine/cache/ChunkCacheMetrics.java | 4 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 2 +-
.../cache/TimeSeriesMetadataCacheMetrics.java | 4 +-
.../apache/iotdb/db/engine/flush/FlushManager.java | 2 +-
.../iotdb/db/engine/flush/FlushManagerMetrics.java | 4 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 6 +-
.../iotdb/db/engine/memtable/AbstractMemTable.java | 6 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 2 +-
.../db/engine/storagegroup/DataRegionMetrics.java | 4 +-
.../engine/storagegroup/TsFileProcessorInfo.java | 2 +-
.../storagegroup/TsFileProcessorInfoMetrics.java | 6 +-
.../iotdb/db/exception/IntoProcessException.java | 16 +-
.../db/metadata/cache/DataNodeSchemaCache.java | 2 +-
.../metadata/cache/DataNodeSchemaCacheMetrics.java | 4 +-
.../db/metadata/mtree/MTreeBelowSGCachedImpl.java | 247 ++++----
.../db/metadata/mtree/MTreeBelowSGMemoryImpl.java | 128 +++--
.../db/metadata/rescon/SchemaResourceManager.java | 2 +-
.../rescon/SchemaStatisticsManagerMetrics.java | 4 +-
.../apache/iotdb/db/mpp/common/SessionInfo.java | 19 +
.../execution/exchange/MPPDataExchangeService.java | 2 +-
.../exchange/MPPDataExchangeServiceMetrics.java | 4 +-
.../MPPDataExchangeServiceThriftHandler.java | 2 +-
...MppDataExchangeServiceThriftHandlerMetrics.java | 6 +-
.../fragment/FragmentInstanceContext.java | 29 +-
.../fragment/FragmentInstanceManager.java | 6 +-
.../db/mpp/execution/operator/OperatorContext.java | 5 +
.../operator/process/AbstractIntoOperator.java | 384 +++++++++++++
.../operator/process/DeviceViewIntoOperator.java | 140 +++++
.../execution/operator/process/IntoOperator.java | 104 ++++
.../process/RawDataAggregationOperator.java | 2 +-
.../process/SingleInputAggregationOperator.java | 2 -
.../process/SlidingWindowAggregationOperator.java | 2 +-
.../schema/TimeSeriesSchemaScanOperator.java | 9 +-
.../execution/schedule/AbstractDriverThread.java | 60 +-
.../db/mpp/execution/schedule/DriverScheduler.java | 38 +-
.../mpp/execution/schedule/DriverTaskThread.java | 5 +-
.../schedule/DriverTaskTimeoutSentinelThread.java | 5 +-
.../execution/schedule/ThreadProducer.java} | 19 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeVisitor.java | 8 +-
.../config/executor/ClusterConfigTaskExecutor.java | 113 +++-
.../config/executor/IConfigTaskExecutor.java | 8 +-
.../executor/StandaloneConfigTaskExecutor.java | 21 +-
.../config/metadata/CreateFunctionTask.java | 15 +-
.../config/metadata/ShowFunctionsTask.java | 155 ++----
.../iotdb/db/mpp/plan/parser/ASTVisitor.java | 59 +-
.../db/mpp/plan/planner/LogicalPlanBuilder.java | 40 ++
.../db/mpp/plan/planner/LogicalPlanVisitor.java | 7 +
.../db/mpp/plan/planner/OperatorTreeGenerator.java | 123 ++++
.../planner/distribution/ExchangeNodeAdder.java | 6 +-
.../SimpleFragmentParallelPlanner.java | 1 +
.../plan/planner/distribution/SourceRewriter.java | 11 +-
.../distribution/WriteFragmentParallelPlanner.java | 3 +-
.../db/mpp/plan/planner/plan/FragmentInstance.java | 22 +-
.../mpp/plan/planner/plan/node/PlanNodeType.java | 10 +-
.../db/mpp/plan/planner/plan/node/PlanVisitor.java | 10 +
.../metedata/read/SchemaQueryOrderByHeatNode.java | 4 +-
.../planner/plan/node/process/AggregationNode.java | 23 +-
.../planner/plan/node/process/DeviceMergeNode.java | 32 +-
.../{TimeJoinNode.java => DeviceViewIntoNode.java} | 95 ++--
.../planner/plan/node/process/DeviceViewNode.java | 19 +-
.../planner/plan/node/process/ExchangeNode.java | 41 +-
.../plan/planner/plan/node/process/FillNode.java | 46 +-
.../plan/node/process/GroupByLevelNode.java | 29 +-
.../planner/plan/node/process/GroupByTagNode.java | 17 +-
.../node/process/{LimitNode.java => IntoNode.java} | 94 ++--
.../plan/planner/plan/node/process/LimitNode.java | 37 +-
...tiChildNode.java => MultiChildProcessNode.java} | 23 +-
.../plan/planner/plan/node/process/OffsetNode.java | 36 +-
.../planner/plan/node/process/ProjectNode.java | 40 +-
...iChildNode.java => SingleChildProcessNode.java} | 50 +-
.../node/process/SlidingWindowAggregationNode.java | 36 +-
.../plan/planner/plan/node/process/SortNode.java | 29 +-
.../planner/plan/node/process/TimeJoinNode.java | 17 +-
.../planner/plan/node/process/TransformNode.java | 29 +-
.../node/process/last/LastQueryCollectNode.java | 4 +-
.../plan/node/process/last/LastQueryMergeNode.java | 4 +-
.../plan/node/process/last/LastQueryNode.java | 4 +-
.../parameter/DeviceViewIntoPathDescriptor.java | 11 +-
.../planner/plan/parameter/IntoPathDescriptor.java | 10 +-
.../plan/scheduler/load/LoadTsFileScheduler.java | 6 +-
.../plan/statement/component/IntoComponent.java | 8 +-
.../db/mpp/plan/statement/crud/QueryStatement.java | 19 +-
.../metadata/CreateFunctionStatement.java | 21 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 22 +-
.../db/query/pool/RawQueryReadTaskPoolManager.java | 2 +-
.../pool/RawQueryReadTaskPoolManagerMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/DataNode.java | 128 ++++-
.../db/service/DataNodeInternalRPCService.java | 2 +-
.../service/DataNodeInternalRPCServiceMetrics.java | 4 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 5 +-
.../java/org/apache/iotdb/db/service/NewIoTDB.java | 5 +-
.../org/apache/iotdb/db/service/RPCService.java | 2 +-
.../apache/iotdb/db/service/RPCServiceMetrics.java | 4 +-
.../db/service/ResourcesInformationHolder.java | 12 +
...erviceMBean.java => DataNodeMetricsHelper.java} | 25 +-
.../metrics/{predefined => }/FileMetrics.java | 6 +-
.../metrics/{predefined => }/ProcessMetrics.java | 6 +-
.../metrics/{predefined => }/SystemMetrics.java | 6 +-
.../metrics/recorder/CacheMetricsRecorder.java | 6 +-
.../recorder/CompactionMetricsRecorder.java | 6 +-
.../db/service/thrift/ProcessorWithMetrics.java | 6 +-
.../handler/InternalServiceThriftHandler.java | 2 +-
.../InternalServiceThriftHandlerMetrics.java | 4 +-
.../thrift/handler/RPCServiceThriftHandler.java | 2 +-
.../handler/RPCServiceThriftHandlerMetrics.java | 4 +-
.../service/thrift/impl/ClientRPCServiceImpl.java | 4 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 32 +-
.../db/service/thrift/impl/TSServiceImpl.java | 4 +-
.../trigger/service/TriggerManagementService.java | 18 +-
.../DriverTaskTimeoutSentinelThreadTest.java | 23 +-
.../iotdb/db/mpp/plan/analyze/AnalyzeFailTest.java | 87 ++-
.../iotdb/db/mpp/plan/analyze/AnalyzeTest.java | 8 +-
.../mpp/plan/plan/FragmentInstanceSerdeTest.java | 10 +-
.../iotdb/db/mpp/plan/plan/LogicalPlannerTest.java | 40 ++
.../plan/plan/node/process/IntoNodeSerdeTest.java | 83 +++
.../plan/scheduler/StandaloneSchedulerTest.java | 18 +-
.../iotdb/db/qp/physical/PhysicalPlanTest.java | 8 +-
.../query/dataset/UDTFAlignByTimeDataSetTest.java | 2 +
.../test/resources/datanode1conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode2conf/iotdb-metric.yml | 7 +-
.../test/resources/datanode3conf/iotdb-metric.yml | 7 +-
server/src/test/resources/logback-test.xml | 2 +-
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 5 +
.../java/org/apache/iotdb/session/ISession.java | 4 +-
.../java/org/apache/iotdb/session/Session.java | 258 ++++++---
.../org/apache/iotdb/session/SessionConfig.java | 2 +-
.../org/apache/iotdb/session/pool/SessionPool.java | 48 +-
.../apache/iotdb/session/SessionCacheLeaderUT.java | 4 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 2 +-
.../java/org/apache/iotdb/db/sql/ClusterIT.java | 2 +-
.../src/main/thrift/confignode.thrift | 47 +-
.../src/main/thrift/mutlileader.thrift | 10 +
thrift/src/main/thrift/datanode.thrift | 16 +-
.../org/apache/iotdb/tsfile/read/common/Path.java | 72 ++-
.../java/org/apache/iotdb/tsfile/utils/BitMap.java | 6 +
250 files changed, 6014 insertions(+), 1972 deletions(-)
diff --cc consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
index 5e2cb5fb7e,911b79c232..dc4ee7007b
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
@@@ -45,10 -50,9 +50,11 @@@ import org.apache.iotdb.consensus.multi
import org.apache.iotdb.consensus.multileader.thrift.TSendSnapshotFragmentRes;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadReq;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadRes;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteReq;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteRes;
import org.apache.iotdb.consensus.multileader.wal.ConsensusReqReader;
import org.apache.iotdb.consensus.multileader.wal.GetConsensusReqReaderPlan;
+ import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.utils.PublicBAOS;
diff --cc thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
index 81252ce353,f85c79c625..0336c19303
--- a/thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
+++ b/thrift-multi-leader-consensus/src/main/thrift/mutlileader.thrift
@@@ -110,7 -109,7 +119,8 @@@ service MultiLeaderConsensusIService
TActivatePeerRes activatePeer(TActivatePeerReq req)
TBuildSyncLogChannelRes buildSyncLogChannel(TBuildSyncLogChannelReq req)
TRemoveSyncLogChannelRes removeSyncLogChannel(TRemoveSyncLogChannelReq req)
+ TWaitSyncLogCompleteRes waitSyncLogComplete(TWaitSyncLogCompleteReq req)
TSendSnapshotFragmentRes sendSnapshotFragment(TSendSnapshotFragmentReq req)
TTriggerSnapshotLoadRes triggerSnapshotLoad(TTriggerSnapshotLoadReq req)
+ TCleanupTransferredSnapshotRes cleanupTransferredSnapshot(TCleanupTransferredSnapshotReq req)
}
[iotdb] 01/03: tmp save
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/migration_add_wait_sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit c5098b4a70465de8fa1f12f3b533c0c8601eaa4d
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Wed Oct 26 16:26:06 2022 +0800
tmp save
---
.../service/MultiLeaderRPCServiceProcessor.java | 9 +++++++++
.../org/apache/iotdb/db/mpp/plan/TestRPCClient.java | 21 ++++++++++++++++++---
2 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
index 20b6aad421..4592756100 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
@@ -238,6 +238,15 @@ public class MultiLeaderRPCServiceProcessor implements MultiLeaderConsensusIServ
ConsensusGroupId groupId =
ConsensusGroupId.Factory.createFromTConsensusGroupId(req.getConsensusGroupId());
MultiLeaderServerImpl impl = consensus.getImpl(groupId);
+ if (impl == null) {
+ String message =
+ String.format("unexpected consensusGroupId %s for waitSyncLogComplete request", groupId);
+ logger.error(message);
+ TSStatus status = new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ status.setMessage(message);
+ resultHandler.onComplete(new TWaitSyncLogCompleteRes(true, 0, 0));
+ return;
+ }
long searchIndex = impl.getIndex();
long safeIndex = impl.getCurrentSafelyDeletedSearchIndex();
resultHandler.onComplete(
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/TestRPCClient.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/TestRPCClient.java
index 4a5b4169fe..cda96d3379 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/TestRPCClient.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/TestRPCClient.java
@@ -33,6 +33,8 @@ import org.apache.iotdb.consensus.multileader.thrift.TInactivatePeerReq;
import org.apache.iotdb.consensus.multileader.thrift.TInactivatePeerRes;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadReq;
import org.apache.iotdb.consensus.multileader.thrift.TTriggerSnapshotLoadRes;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteReq;
+import org.apache.iotdb.consensus.multileader.thrift.TWaitSyncLogCompleteRes;
import org.apache.iotdb.db.client.DataNodeClientPoolFactory;
import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq;
@@ -62,11 +64,24 @@ public class TestRPCClient {
public static void main(String args[]) {
TestRPCClient client = new TestRPCClient();
- // client.removeRegionPeer();
- client.addPeer();
+ client.removeRegionPeer();
+// client.testWaitSyncLog();
// client.loadSnapshot();
}
+ private void testWaitSyncLog() {
+ try (SyncMultiLeaderServiceClient client =
+ syncClientManager.borrowClient(new TEndPoint("127.0.0.1", 40012))) {
+ TWaitSyncLogCompleteRes res =
+ client.waitSyncLogComplete(
+ new TWaitSyncLogCompleteReq(new DataRegionId(1).convertToTConsensusGroupId()));
+ System.out.printf("%s, %d, %d",res.complete, res.searchIndex, res.safeIndex);
+ } catch (IOException | TException e) {
+ System.out.println("Error: " + e.getMessage());
+ throw new RuntimeException(e);
+ }
+ }
+
private void loadSnapshot() {
try (SyncMultiLeaderServiceClient client =
syncClientManager.borrowClient(new TEndPoint("127.0.0.1", 40011))) {
@@ -96,7 +111,7 @@ public class TestRPCClient {
try (SyncDataNodeInternalServiceClient client =
INTERNAL_SERVICE_CLIENT_MANAGER.borrowClient(new TEndPoint("127.0.0.1", 9003))) {
client.removeRegionPeer(
- new TMaintainPeerReq(new DataRegionId(1).convertToTConsensusGroupId(), getLocation2(3)));
+ new TMaintainPeerReq(new DataRegionId(1).convertToTConsensusGroupId(), getLocation3(3)));
} catch (IOException | TException e) {
throw new RuntimeException(e);
}
[iotdb] 03/03: complete wait logic when migrate dataregion for ML
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch xingtanzjr/migration_add_wait_sync
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 74c5ee4b7d94051e08351236ba1a0cad6edc0e7c
Author: Jinrui.Zhang <xi...@gmail.com>
AuthorDate: Thu Oct 27 16:47:10 2022 +0800
complete wait logic when migrate dataregion for ML
---
.../apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
index dc4ee7007b..80957fb29a 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
@@ -491,6 +491,11 @@ public class MultiLeaderServerImpl {
client.waitSyncLogComplete(
new TWaitSyncLogCompleteReq(targetPeer.getGroupId().convertToTConsensusGroupId()));
if (res.complete) {
+ logger.info(
+ "{} SyncLog is completed. TargetIndex: {}, CurrentSyncIndex: {}",
+ targetPeer,
+ res.searchIndex,
+ res.safeIndex);
return;
}
logger.info(