You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2025/09/28 04:05:13 UTC
(iotdb) branch master updated (4bdba5c953d -> 49e857287a6)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 4bdba5c953d Pipe: Reduced the conversion logger & Fixed the illegal formats of PipeLogger (#16503)
new 8160a18727a reformat procedure
new 49e857287a6 spotless procedure
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(iotdb) 02/02: spotless procedure
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 49e857287a6f0058b530550fcb387454e789bb7c
Author: Yongzao <53...@qq.com>
AuthorDate: Sun Sep 28 12:03:11 2025 +0800
spotless procedure
---
.../iotdb/confignode/manager/ProcedureManager.java | 2000 ++++++++++----------
.../procedure/CompletedProcedureContainer.java | 73 +-
.../procedure/CompletedProcedureRecycler.java | 169 +-
.../confignode/procedure/InternalProcedure.java | 99 +-
.../procedure/PartitionTableAutoCleaner.java | 225 ++-
.../iotdb/confignode/procedure/Procedure.java | 1647 ++++++++--------
.../confignode/procedure/ProcedureExecutor.java | 1881 +++++++++---------
.../confignode/procedure/ProcedureMetrics.java | 369 ++--
.../confignode/procedure/RootProcedureStack.java | 355 ++--
.../confignode/procedure/StoppableThread.java | 111 +-
.../procedure/TimeoutExecutorThread.java | 215 ++-
.../procedure/env/ConfigNodeProcedureEnv.java | 1789 +++++++++--------
.../procedure/env/RegionMaintainHandler.java | 1175 ++++++------
.../procedure/env/RemoveDataNodeHandler.java | 1363 +++++++------
.../procedure/exception/ProcedureException.java | 89 +-
.../scheduler/AbstractProcedureScheduler.java | 441 +++--
.../confignode/procedure/scheduler/LockQueue.java | 119 +-
.../procedure/scheduler/ProcedureScheduler.java | 223 ++-
.../scheduler/SimpleProcedureScheduler.java | 159 +-
.../procedure/store/ConfigProcedureStore.java | 363 ++--
.../procedure/store/IProcedureFactory.java | 61 +-
.../procedure/store/IProcedureStore.java | 117 +-
.../procedure/store/ProcedureFactory.java | 1119 ++++++-----
.../confignode/procedure/store/ProcedureType.java | 377 ++--
.../confignode/procedure/store/ProcedureWAL.java | 153 +-
25 files changed, 7334 insertions(+), 7358 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 96ee4e1c680..3dcb01de273 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -1,1003 +1,1003 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.manager;
-1
-1import org.apache.iotdb.common.rpc.thrift.Model;
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-1import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-1import org.apache.iotdb.common.rpc.thrift.TSStatus;
-1import org.apache.iotdb.commons.cluster.NodeStatus;
-1import org.apache.iotdb.commons.conf.CommonConfig;
-1import org.apache.iotdb.commons.conf.CommonDescriptor;
-1import org.apache.iotdb.commons.conf.IoTDBConstant;
-1import org.apache.iotdb.commons.exception.IoTDBException;
-1import org.apache.iotdb.commons.path.PartialPath;
-1import org.apache.iotdb.commons.path.PathDeserializeUtil;
-1import org.apache.iotdb.commons.path.PathPatternTree;
-1import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
-1import org.apache.iotdb.commons.schema.table.TsTable;
-1import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
-1import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
-1import org.apache.iotdb.commons.service.metric.MetricService;
-1import org.apache.iotdb.commons.trigger.TriggerInformation;
-1import org.apache.iotdb.commons.utils.StatusUtils;
-1import org.apache.iotdb.commons.utils.TestOnly;
-1import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
-1import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
-1import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan;
-1import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
-1import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
-1import org.apache.iotdb.confignode.manager.partition.PartitionManager;
-1import org.apache.iotdb.confignode.persistence.ProcedureInfo;
-1import org.apache.iotdb.confignode.procedure.PartitionTableAutoCleaner;
-1import org.apache.iotdb.confignode.procedure.Procedure;
-1import org.apache.iotdb.confignode.procedure.ProcedureExecutor;
-1import org.apache.iotdb.confignode.procedure.ProcedureMetrics;
-1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-1import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler;
-1import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler;
-1import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleLeaderChangeProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeMetaSyncProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.task.AlterPipeProcedureV2;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.task.CreatePipeProcedureV2;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.task.StartPipeProcedureV2;
-1import org.apache.iotdb.confignode.procedure.impl.pipe.task.StopPipeProcedureV2;
-1import org.apache.iotdb.confignode.procedure.impl.region.AddRegionPeerProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan;
-1import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.AddViewColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.CreateTableViewProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewColumnProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.SetViewPropertiesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.CreateSubscriptionProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.DropSubscriptionProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.CreateTopicProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.DropTopicProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.runtime.TopicMetaSyncProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.sync.AuthOperationProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.testonly.AddNeverFinishSubProcedureProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.testonly.CreateManyDatabasesProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.trigger.CreateTriggerProcedure;
-1import org.apache.iotdb.confignode.procedure.impl.trigger.DropTriggerProcedure;
-1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-1import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
-1import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore;
-1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-1import org.apache.iotdb.confignode.procedure.store.ProcedureFactory;
-1import org.apache.iotdb.confignode.procedure.store.ProcedureType;
-1import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TCreateTopicReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
-1import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
-1import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq;
-1import org.apache.iotdb.consensus.ConsensusFactory;
-1import org.apache.iotdb.db.exception.BatchProcessException;
-1import org.apache.iotdb.db.schemaengine.template.Template;
-1import org.apache.iotdb.db.utils.constant.SqlConstant;
-1import org.apache.iotdb.rpc.RpcUtils;
-1import org.apache.iotdb.rpc.TSStatusCode;
-1
-1import org.apache.ratis.util.AutoCloseableLock;
-1import org.apache.tsfile.utils.Binary;
-1import org.apache.tsfile.utils.Pair;
-1import org.apache.tsfile.utils.ReadWriteIOUtils;
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import javax.annotation.Nonnull;
-1import javax.annotation.Nullable;
-1
-1import java.io.IOException;
-1import java.nio.ByteBuffer;
-1import java.util.ArrayList;
-1import java.util.Arrays;
-1import java.util.HashMap;
-1import java.util.List;
-1import java.util.Map;
-1import java.util.Objects;
-1import java.util.Optional;
-1import java.util.Set;
-1import java.util.concurrent.ScheduledExecutorService;
-1import java.util.concurrent.locks.ReentrantLock;
-1import java.util.function.BiFunction;
-1import java.util.stream.Collectors;
-1import java.util.stream.Stream;
-1
-1public class ProcedureManager {
-1 private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureManager.class);
-1
-1 private static final ConfigNodeConfig CONFIG_NODE_CONFIG =
-1 ConfigNodeDescriptor.getInstance().getConf();
-1 private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
-1
-1 public static final long PROCEDURE_WAIT_TIME_OUT = COMMON_CONFIG.getDnConnectionTimeoutInMS();
-1 public static final int PROCEDURE_WAIT_RETRY_TIMEOUT = 10;
-1 private static final String PROCEDURE_TIMEOUT_MESSAGE =
-1 "Timed out to wait for procedure return. The procedure is still running.";
-1
-1 private final ConfigManager configManager;
-1 private ProcedureExecutor<ConfigNodeProcedureEnv> executor;
-1 private ProcedureScheduler scheduler;
-1 private IProcedureStore store;
-1 private ConfigNodeProcedureEnv env;
-1
-1 private final long planSizeLimit;
-1 private ProcedureMetrics procedureMetrics;
-1
-1 private final PartitionTableAutoCleaner partitionTableCleaner;
-1
-1 private final ReentrantLock tableLock = new ReentrantLock();
-1
-1 public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo) {
-1 this.configManager = configManager;
-1 this.scheduler = new SimpleProcedureScheduler();
-1 this.store = new ConfigProcedureStore(configManager, procedureInfo);
-1 this.env = new ConfigNodeProcedureEnv(configManager, scheduler);
-1 this.executor = new ProcedureExecutor<>(env, store, scheduler);
-1 this.planSizeLimit =
-1 ConfigNodeDescriptor.getInstance()
-1 .getConf()
-1 .getConfigNodeRatisConsensusLogAppenderBufferSize()
-1 - IoTDBConstant.RAFT_LOG_BASIC_SIZE;
-1 this.procedureMetrics = new ProcedureMetrics(this);
-1 this.partitionTableCleaner = new PartitionTableAutoCleaner<>(configManager);
-1 }
-1
-1 public void startExecutor() {
-1 if (!executor.isRunning()) {
-1 executor.init(CONFIG_NODE_CONFIG.getProcedureCoreWorkerThreadsCount());
-1 executor.startWorkers();
-1 executor.startCompletedCleaner(
-1 CONFIG_NODE_CONFIG.getProcedureCompletedCleanInterval(),
-1 CONFIG_NODE_CONFIG.getProcedureCompletedEvictTTL());
-1 executor.addInternalProcedure(partitionTableCleaner);
-1 store.start();
-1 LOGGER.info("ProcedureManager is started successfully.");
-1 }
-1 }
-1
-1 public void stopExecutor() {
-1 if (executor.isRunning()) {
-1 executor.stop();
-1 if (!executor.isRunning()) {
-1 executor.join();
-1 store.stop();
-1 LOGGER.info("ProcedureManager is stopped successfully.");
-1 }
-1 executor.removeInternalProcedure(partitionTableCleaner);
-1 }
-1 }
-1
-1 @TestOnly
-1 public TSStatus createManyDatabases() {
-1 this.executor.submitProcedure(new CreateManyDatabasesProcedure());
-1 return StatusUtils.OK;
-1 }
-1
-1 @TestOnly
-1 public TSStatus testSubProcedure() {
-1 this.executor.submitProcedure(new AddNeverFinishSubProcedureProcedure());
-1 return StatusUtils.OK;
-1 }
-1
-1 public TSStatus deleteDatabases(
-1 final List<TDatabaseSchema> deleteSgSchemaList, final boolean isGeneratedByPipe) {
-1 final List<DeleteDatabaseProcedure> procedures = new ArrayList<>();
-1 final long startCheckTimeForProcedures = System.currentTimeMillis();
-1 for (final TDatabaseSchema databaseSchema : deleteSgSchemaList) {
-1 final String database = databaseSchema.getName();
-1 boolean hasOverlappedTask = false;
-1 synchronized (this) {
-1 while (executor.isRunning()
-1 && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) {
-1 final Pair<Long, Boolean> procedureIdDuplicatePair =
-1 checkDuplicateTableTask(
-1 database, null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE);
-1 hasOverlappedTask = procedureIdDuplicatePair.getRight();
-1
-1 if (Boolean.FALSE.equals(procedureIdDuplicatePair.getRight())) {
-1 DeleteDatabaseProcedure procedure =
-1 new DeleteDatabaseProcedure(databaseSchema, isGeneratedByPipe);
-1 this.executor.submitProcedure(procedure);
-1 procedures.add(procedure);
-1 break;
-1 }
-1 try {
-1 wait(PROCEDURE_WAIT_RETRY_TIMEOUT);
-1 } catch (final InterruptedException e) {
-1 Thread.currentThread().interrupt();
-1 }
-1 }
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 String.format(
-1 "Some other task is operating table under the database %s, please retry after the procedure finishes.",
-1 database));
-1 }
-1 }
-1 }
-1 List<TSStatus> results = new ArrayList<>(procedures.size());
-1 procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure)));
-1 // Clear the previously deleted regions
-1 final PartitionManager partitionManager = getConfigManager().getPartitionManager();
-1 partitionManager.getRegionMaintainer().submit(partitionManager::maintainRegionReplicas);
-1 if (results.stream()
-1 .allMatch(result -> result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode())) {
-1 return StatusUtils.OK;
-1 } else {
-1 return RpcUtils.getStatus(results);
-1 }
-1 }
-1
-1 public TSStatus deleteTimeSeries(
-1 String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) {
-1 DeleteTimeSeriesProcedure procedure = null;
-1 synchronized (this) {
-1 boolean hasOverlappedTask = false;
-1 ProcedureType type;
-1 DeleteTimeSeriesProcedure deleteTimeSeriesProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.DELETE_TIMESERIES_PROCEDURE)) {
-1 continue;
-1 }
-1 deleteTimeSeriesProcedure = ((DeleteTimeSeriesProcedure) runningProcedure);
-1 if (queryId.equals(deleteTimeSeriesProcedure.getQueryId())) {
-1 procedure = deleteTimeSeriesProcedure;
-1 break;
-1 }
-1 if (patternTree.isOverlapWith(deleteTimeSeriesProcedure.getPatternTree())) {
-1 hasOverlappedTask = true;
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 "Some other task is deleting some target timeseries.");
-1 }
-1 procedure = new DeleteTimeSeriesProcedure(queryId, patternTree, isGeneratedByPipe);
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 public TSStatus deleteLogicalView(TDeleteLogicalViewReq req) {
-1 String queryId = req.getQueryId();
-1 PathPatternTree patternTree =
-1 PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
-1 DeleteLogicalViewProcedure procedure = null;
-1 synchronized (this) {
-1 boolean hasOverlappedTask = false;
-1 ProcedureType type;
-1 DeleteLogicalViewProcedure deleteLogicalViewProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.DELETE_LOGICAL_VIEW_PROCEDURE)) {
-1 continue;
-1 }
-1 deleteLogicalViewProcedure = ((DeleteLogicalViewProcedure) runningProcedure);
-1 if (queryId.equals(deleteLogicalViewProcedure.getQueryId())) {
-1 procedure = deleteLogicalViewProcedure;
-1 break;
-1 }
-1 if (patternTree.isOverlapWith(deleteLogicalViewProcedure.getPatternTree())) {
-1 hasOverlappedTask = true;
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 "Some other task is deleting some target views.");
-1 }
-1 procedure =
-1 new DeleteLogicalViewProcedure(
-1 queryId, patternTree, req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 public TSStatus alterLogicalView(final TAlterLogicalViewReq req) {
-1 final String queryId = req.getQueryId();
-1 final ByteBuffer byteBuffer = ByteBuffer.wrap(req.getViewBinary());
-1 final Map<PartialPath, ViewExpression> viewPathToSourceMap = new HashMap<>();
-1 final int size = byteBuffer.getInt();
-1 PartialPath path;
-1 ViewExpression viewExpression;
-1 for (int i = 0; i < size; i++) {
-1 path = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer);
-1 viewExpression = ViewExpression.deserialize(byteBuffer);
-1 viewPathToSourceMap.put(path, viewExpression);
-1 }
-1
-1 AlterLogicalViewProcedure procedure = null;
-1 synchronized (this) {
-1 ProcedureType type;
-1 AlterLogicalViewProcedure alterLogicalViewProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.ALTER_LOGICAL_VIEW_PROCEDURE)) {
-1 continue;
-1 }
-1 alterLogicalViewProcedure = ((AlterLogicalViewProcedure) runningProcedure);
-1 if (queryId.equals(alterLogicalViewProcedure.getQueryId())) {
-1 procedure = alterLogicalViewProcedure;
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 procedure =
-1 new AlterLogicalViewProcedure(
-1 queryId,
-1 viewPathToSourceMap,
-1 req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 public TSStatus setSchemaTemplate(
-1 String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) {
-1 SetTemplateProcedure procedure = null;
-1 synchronized (this) {
-1 boolean hasOverlappedTask = false;
-1 ProcedureType type;
-1 SetTemplateProcedure setTemplateProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
-1 continue;
-1 }
-1 setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
-1 if (queryId.equals(setTemplateProcedure.getQueryId())) {
-1 procedure = setTemplateProcedure;
-1 break;
-1 }
-1 if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
-1 hasOverlappedTask = true;
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 "Some other task is setting template on target path.");
-1 }
-1 procedure =
-1 new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe);
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 public TSStatus deactivateTemplate(
-1 String queryId, Map<PartialPath, List<Template>> templateSetInfo, boolean isGeneratedByPipe) {
-1 DeactivateTemplateProcedure procedure = null;
-1 synchronized (this) {
-1 boolean hasOverlappedTask = false;
-1 ProcedureType type;
-1 DeactivateTemplateProcedure deactivateTemplateProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.DEACTIVATE_TEMPLATE_PROCEDURE)) {
-1 continue;
-1 }
-1 deactivateTemplateProcedure = (DeactivateTemplateProcedure) runningProcedure;
-1 if (queryId.equals(deactivateTemplateProcedure.getQueryId())) {
-1 procedure = deactivateTemplateProcedure;
-1 break;
-1 }
-1 for (PartialPath pattern : templateSetInfo.keySet()) {
-1 for (PartialPath existingPattern :
-1 deactivateTemplateProcedure.getTemplateSetInfo().keySet()) {
-1 if (pattern.overlapWith(existingPattern)) {
-1 hasOverlappedTask = true;
-1 break;
-1 }
-1 }
-1 if (hasOverlappedTask) {
-1 break;
-1 }
-1 }
-1 if (hasOverlappedTask) {
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 "Some other task is deactivating some target template from target path.");
-1 }
-1 procedure = new DeactivateTemplateProcedure(queryId, templateSetInfo, isGeneratedByPipe);
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 public TSStatus unsetSchemaTemplate(
-1 String queryId, Template template, PartialPath path, boolean isGeneratedByPipe) {
-1 UnsetTemplateProcedure procedure = null;
-1 synchronized (this) {
-1 boolean hasOverlappedTask = false;
-1 ProcedureType type;
-1 UnsetTemplateProcedure unsetTemplateProcedure;
-1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
-1 type = ProcedureFactory.getProcedureType(runningProcedure);
-1 if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
-1 continue;
-1 }
-1 unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
-1 if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
-1 procedure = unsetTemplateProcedure;
-1 break;
-1 }
-1 if (template.getId() == unsetTemplateProcedure.getTemplateId()
-1 && path.equals(unsetTemplateProcedure.getPath())) {
-1 hasOverlappedTask = true;
-1 break;
-1 }
-1 }
-1
-1 if (procedure == null) {
-1 if (hasOverlappedTask) {
-1 return RpcUtils.getStatus(
-1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
-1 "Some other task is unsetting target template from target path "
-1 + path.getFullPath());
-1 }
-1 procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe);
-1 this.executor.submitProcedure(procedure);
-1 }
-1 }
-1 return waitingProcedureFinished(procedure);
-1 }
-1
-1 /**
-1 * Generate an {@link AddConfigNodeProcedure}, and serially execute all the {@link
-1 * AddConfigNodeProcedure}s.
-1 */
-1 public void addConfigNode(TConfigNodeRegisterReq req) {
-1 final AddConfigNodeProcedure addConfigNodeProcedure =
-1 new AddConfigNodeProcedure(req.getConfigNodeLocation(), req.getVersionInfo());
-1 this.executor.submitProcedure(addConfigNodeProcedure);
-1 }
-1
-1 /**
-1 * Generate a {@link RemoveConfigNodeProcedure}, and serially execute all the {@link
-1 * RemoveConfigNodeProcedure}s.
-1 */
-1 public void removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) {
-1 final RemoveConfigNodeProcedure removeConfigNodeProcedure =
-1 new RemoveConfigNodeProcedure(removeConfigNodePlan.getConfigNodeLocation());
-1 this.executor.submitProcedure(removeConfigNodeProcedure);
-1 LOGGER.info("Submit RemoveConfigNodeProcedure successfully: {}", removeConfigNodePlan);
-1 }
-1
-1 /**
-1 * Generate {@link RemoveDataNodesProcedure}s, and serially execute all the {@link
-1 * RemoveDataNodesProcedure}s.
-1 */
-1 public boolean removeDataNode(RemoveDataNodePlan removeDataNodePlan) {
-1 Map<Integer, NodeStatus> nodeStatusMap = new HashMap<>();
-1 removeDataNodePlan
-1 .getDataNodeLocations()
-1 .forEach(
-1 datanode ->
-1 nodeStatusMap.put(
-1 datanode.getDataNodeId(),
-1 configManager.getLoadManager().getNodeStatus(datanode.getDataNodeId())));
-1 this.executor.submitProcedure(
-1 new RemoveDataNodesProcedure(removeDataNodePlan.getDataNodeLocations(), nodeStatusMap));
-1 LOGGER.info(
-1 "Submit RemoveDataNodesProcedure successfully, {}",
-1 removeDataNodePlan.getDataNodeLocations());
-1 return true;
-1 }
-1
-1 public boolean removeAINode(RemoveAINodePlan removeAINodePlan) {
-1 this.executor.submitProcedure(new RemoveAINodeProcedure(removeAINodePlan.getAINodeLocation()));
-1 LOGGER.info(
-1 "Submit RemoveAINodeProcedure successfully, {}", removeAINodePlan.getAINodeLocation());
-1 return true;
-1 }
-1
-1 public TSStatus checkRemoveDataNodes(List<TDataNodeLocation> dataNodeLocations) {
-1 // 1. Only one RemoveDataNodesProcedure is allowed in the cluster
-1 Optional<Procedure<ConfigNodeProcedureEnv>> anotherRemoveProcedure =
-1 getExecutor().getProcedures().values().stream()
-1 .filter(
-1 procedure -> {
-1 if (procedure instanceof RemoveDataNodesProcedure) {
-1 return !procedure.isFinished();
-1 }
-1 return false;
-1 })
-1 .findAny();
-1
-1 String failMessage = null;
-1 if (anotherRemoveProcedure.isPresent()) {
-1 List<TDataNodeLocation> anotherRemoveDataNodes =
-1 ((RemoveDataNodesProcedure) anotherRemoveProcedure.get()).getRemovedDataNodes();
-1 failMessage =
-1 String.format(
-1 "Submit RemoveDataNodesProcedure failed, "
-1 + "because another RemoveDataNodesProcedure %s is already in processing. "
-1 + "IoTDB is able to have at most 1 RemoveDataNodesProcedure at the same time. "
-1 + "For further information, please search [pid%d] in log. ",
-1 anotherRemoveDataNodes, anotherRemoveProcedure.get().getProcId());
-1 }
-1
-1 // 2. Check if the RemoveDataNodesProcedure conflicts with the RegionMigrateProcedure
-1 Set<TConsensusGroupId> removedDataNodesRegionSet =
-1 getEnv().getRemoveDataNodeHandler().getRemovedDataNodesRegionSet(dataNodeLocations);
-1 Optional<Procedure<ConfigNodeProcedureEnv>> conflictRegionMigrateProcedure =
-1 getExecutor().getProcedures().values().stream()
-1 .filter(
-1 procedure -> {
-1 if (procedure instanceof RegionMigrateProcedure) {
-1 RegionMigrateProcedure regionMigrateProcedure =
-1 (RegionMigrateProcedure) procedure;
-1 if (regionMigrateProcedure.isFinished()) {
-1 return false;
-1 }
-1 return removedDataNodesRegionSet.contains(regionMigrateProcedure.getRegionId())
-1 || dataNodeLocations.contains(regionMigrateProcedure.getDestDataNode());
-1 }
-1 return false;
-1 })
-1 .findAny();
-1 if (conflictRegionMigrateProcedure.isPresent()) {
-1 failMessage =
-1 String.format(
-1 "Submit RemoveDataNodesProcedure failed, "
-1 + "because another RegionMigrateProcedure %s is already in processing which conflicts with this RemoveDataNodesProcedure. "
-1 + "The RegionMigrateProcedure is migrating the region %s to the DataNode %s. "
-1 + "For further information, please search [pid%d] in log. ",
-1 conflictRegionMigrateProcedure.get().getProcId(),
-1 ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getRegionId(),
-1 ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getDestDataNode(),
-1 conflictRegionMigrateProcedure.get().getProcId());
-1 }
-1 // 3. Check if the RegionMigrateProcedure generated by RemoveDataNodesProcedure conflicts with
-1 // each other
-1 List<RegionMigrationPlan> regionMigrationPlans =
-1 getEnv().getRemoveDataNodeHandler().getRegionMigrationPlans(dataNodeLocations);
-1 removedDataNodesRegionSet.clear();
-1 for (RegionMigrationPlan regionMigrationPlan : regionMigrationPlans) {
-1 if (removedDataNodesRegionSet.contains(regionMigrationPlan.getRegionId())) {
-1 failMessage =
-1 String.format(
-1 "Submit RemoveDataNodesProcedure failed, "
-1 + "because the RegionMigrateProcedure generated by this RemoveDataNodesProcedure conflicts with each other. "
-1 + "Only one replica of the same consensus group is allowed to be migrated at the same time."
-1 + "The conflict region id is %s . ",
-1 regionMigrationPlan.getRegionId());
-1 break;
-1 }
-1 removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId());
-1 }
-1
-1 // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are
-1 // not the remove DataNodes
-1
-1 for (TDataNodeLocation removeDataNode : dataNodeLocations) {
-1 Set<TDataNodeLocation> relatedDataNodes =
-1 getEnv().getRemoveDataNodeHandler().getRelatedDataNodeLocations(removeDataNode);
-1 relatedDataNodes.remove(removeDataNode);
-1
-1 for (TDataNodeLocation relatedDataNode : relatedDataNodes) {
-1 NodeStatus nodeStatus =
-1 getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId());
-1 if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) {
-1 failMessage =
-1 String.format(
-1 "Submit RemoveDataNodesProcedure failed, "
-1 + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, "
-1 + "the remove operation cannot be performed for security reasons. "
-1 + "Please check the status of the node %s and ensure it is running.",
-1 relatedDataNode.getDataNodeId());
-1 }
-1 }
-1 }
-1
-1 if (failMessage != null) {
-1 LOGGER.warn(failMessage);
-1 TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_DATANODE_ERROR.getStatusCode());
-1 failStatus.setMessage(failMessage);
-1 return failStatus;
-1 }
-1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 // region region operation related check
-1
-1 /**
-1 * Checks whether region migration is allowed.
-1 *
-1 * @param migrateRegionReq the migration request details
-1 * @param regionGroupId the ID of the consensus group for the region
-1 * @param originalDataNode the original DataNode location from which the region is being migrated
-1 * @param destDataNode the destination DataNode location to which the region is being migrated
-1 * @param coordinatorForAddPeer the DataNode location acting as the coordinator for adding a peer
-1 * @return the status of the migration request (TSStatus)
-1 */
-1 private TSStatus checkMigrateRegion(
-1 TMigrateRegionReq migrateRegionReq,
-1 TConsensusGroupId regionGroupId,
-1 TDataNodeLocation originalDataNode,
-1 TDataNodeLocation destDataNode,
-1 TDataNodeLocation coordinatorForAddPeer) {
-1 String failMessage;
-1 if ((failMessage =
-1 regionOperationCommonCheck(
-1 regionGroupId,
-1 destDataNode,
-1 Arrays.asList(
-1 new Pair<>("Original DataNode", originalDataNode),
-1 new Pair<>("Destination DataNode", destDataNode),
-1 new Pair<>("Coordinator for add peer", coordinatorForAddPeer)),
-1 migrateRegionReq.getModel()))
-1 != null) {
-1 // do nothing
-1 } else if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSets(originalDataNode.getDataNodeId())
-1 .stream()
-1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
-1 failMessage =
-1 String.format(
-1 "Submit RegionMigrateProcedure failed, because the original DataNode %s doesn't contain Region %s",
-1 migrateRegionReq.getFromId(), migrateRegionReq.getRegionId());
-1 } else if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSets(destDataNode.getDataNodeId())
-1 .stream()
-1 .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
-1 failMessage =
-1 String.format(
-1 "Submit RegionMigrateProcedure failed, because the target DataNode %s already contains Region %s",
-1 migrateRegionReq.getToId(), migrateRegionReq.getRegionId());
-1 }
-1
-1 if (failMessage != null) {
-1 LOGGER.warn(failMessage);
-1 TSStatus failStatus = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
-1 failStatus.setMessage(failMessage);
-1 return failStatus;
-1 }
-1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 private TSStatus checkReconstructRegion(
-1 TReconstructRegionReq req,
-1 TConsensusGroupId regionId,
-1 TDataNodeLocation targetDataNode,
-1 TDataNodeLocation coordinator) {
-1 String failMessage =
-1 regionOperationCommonCheck(
-1 regionId,
-1 targetDataNode,
-1 Arrays.asList(
-1 new Pair<>("Target DataNode", targetDataNode),
-1 new Pair<>("Coordinator", coordinator)),
-1 req.getModel());
-1
-1 ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
-1 if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSetsMap(regionId.getType())
-1 .get(regionId)
-1 .getDataNodeLocationsSize()
-1 == 1) {
-1 failMessage = String.format("%s only has 1 replica, it cannot be reconstructed", regionId);
-1 } else if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSets(targetDataNode.getDataNodeId())
-1 .stream()
-1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
-1 failMessage =
-1 String.format(
-1 "Submit ReconstructRegionProcedure failed, because the target DataNode %s doesn't contain Region %s",
-1 req.getDataNodeId(), regionId);
-1 }
-1
-1 if (failMessage != null) {
-1 LOGGER.warn(failMessage);
-1 TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
-1 failStatus.setMessage(failMessage);
-1 return failStatus;
-1 }
-1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 private TSStatus checkExtendRegion(
-1 TExtendRegionReq req,
-1 TConsensusGroupId regionId,
-1 TDataNodeLocation targetDataNode,
-1 TDataNodeLocation coordinator) {
-1 String failMessage =
-1 regionOperationCommonCheck(
-1 regionId,
-1 targetDataNode,
-1 Arrays.asList(
-1 new Pair<>("Target DataNode", targetDataNode),
-1 new Pair<>("Coordinator", coordinator)),
-1 req.getModel());
-1 if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSets(targetDataNode.getDataNodeId())
-1 .stream()
-1 .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
-1 failMessage =
-1 String.format(
-1 "Target DataNode %s already contains region %s",
-1 targetDataNode.getDataNodeId(), regionId);
-1 }
-1
-1 if (failMessage != null) {
-1 LOGGER.warn(failMessage);
-1 TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
-1 failStatus.setMessage(failMessage);
-1 return failStatus;
-1 }
-1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 private TSStatus checkRemoveRegion(
-1 TRemoveRegionReq req,
-1 TConsensusGroupId regionId,
-1 @Nullable TDataNodeLocation targetDataNode,
-1 TDataNodeLocation coordinator) {
-1 String failMessage =
-1 regionOperationCommonCheck(
-1 regionId,
-1 targetDataNode,
-1 Arrays.asList(new Pair<>("Coordinator", coordinator)),
-1 req.getModel());
-1
-1 if (configManager
-1 .getPartitionManager()
-1 .getAllReplicaSetsMap(regionId.getType())
-1 .get(regionId)
-1 .getDataNodeLocationsSize()
-1 == 1) {
-1 failMessage = String.format("%s only has 1 replica, it cannot be removed", regionId);
-1 } else if (targetDataNode != null
-1 && configManager
-1 .getPartitionManager()
-1 .getAllReplicaSets(targetDataNode.getDataNodeId())
-1 .stream()
-1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
-1 failMessage =
-1 String.format(
-1 "Target DataNode %s doesn't contain Region %s", req.getDataNodeId(), regionId);
-1 }
-1
-1 if (failMessage != null) {
-1 LOGGER.warn(failMessage);
-1 TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_REGION_PEER_ERROR.getStatusCode());
-1 failStatus.setMessage(failMessage);
-1 return failStatus;
-1 }
-1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 /**
-1 * The common checks of all region operations, include migration, reconstruction, extension,
-1 * removing
-1 *
-1 * @param regionId region group id, also called consensus group id
-1 * @param targetDataNode DataNode should in Running status
-1 * @param relatedDataNodes Pair<Identity, Node Location>
-1 * @return The reason if check failed, or null if check pass
-1 */
-1 private String regionOperationCommonCheck(
-1 TConsensusGroupId regionId,
-1 TDataNodeLocation targetDataNode,
-1 List<Pair<String, TDataNodeLocation>> relatedDataNodes,
-1 Model model) {
-1 String failMessage;
-1 ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
-1
-1 if (TConsensusGroupType.DataRegion == regionId.getType()
-1 && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) {
-1 failMessage = "SimpleConsensus not supports region operation.";
-1 } else if (TConsensusGroupType.SchemaRegion == regionId.getType()
-1 && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getSchemaRegionConsensusProtocolClass())) {
-1 failMessage = "SimpleConsensus not supports region operation.";
-1 } else if ((failMessage = checkRegionOperationDuplication(regionId)) != null) {
-1 // need to do nothing more
-1 } else if (relatedDataNodes.stream().anyMatch(pair -> pair.getRight() == null)) {
-1 Pair<String, TDataNodeLocation> nullPair =
-1 relatedDataNodes.stream().filter(pair -> pair.getRight() == null).findAny().get();
-1 failMessage = String.format("Cannot find %s", nullPair.getLeft());
-1 } else if (targetDataNode != null
-1 && !configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running).stream()
-1 .map(TDataNodeConfiguration::getLocation)
-1 .map(TDataNodeLocation::getDataNodeId)
-1 .collect(Collectors.toSet())
-1 .contains(targetDataNode.getDataNodeId())) {
-1 // Here we only check Running DataNode to implement migration, because removing nodes may not
-1 // exist when add peer is performing
-1 failMessage =
-1 String.format(
-1 "Target DataNode %s is not in Running status.", targetDataNode.getDataNodeId());
-1 } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode))
-1 != null) {
-1 // need to do nothing more
-1 } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) {
-1 // need to do nothing more
-1 }
-1
-1 return failMessage;
-1 }
-1
-1 private String checkRegionOperationWithRemoveDataNode(
-1 TConsensusGroupId regionId, TDataNodeLocation targetDataNode) {
-1 Optional<Procedure<ConfigNodeProcedureEnv>> conflictRemoveDataNodesProcedure =
-1 getExecutor().getProcedures().values().stream()
-1 .filter(
-1 procedure -> {
-1 if (procedure instanceof RemoveDataNodesProcedure) {
-1 return !procedure.isFinished();
-1 }
-1 return false;
-1 })
-1 .findAny();
-1
-1 if (conflictRemoveDataNodesProcedure.isPresent()) {
-1 RemoveDataNodeHandler removeDataNodeHandler = env.getRemoveDataNodeHandler();
-1 List<TDataNodeLocation> removedDataNodes =
-1 ((RemoveDataNodesProcedure) conflictRemoveDataNodesProcedure.get()).getRemovedDataNodes();
-1 Set<TConsensusGroupId> removedDataNodesRegionSet =
-1 removeDataNodeHandler.getRemovedDataNodesRegionSet(removedDataNodes);
-1 if (removedDataNodesRegionSet.contains(regionId)) {
-1 return String.format(
-1 "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
-1 + "The RemoveDataNodesProcedure is removing the DataNodes %s which contains the region %s. "
-1 + "For further information, please search [pid%d] in log. ",
-1 conflictRemoveDataNodesProcedure.get().getProcId(),
-1 removedDataNodes,
-1 regionId,
-1 conflictRemoveDataNodesProcedure.get().getProcId());
-1 } else if (removedDataNodes.contains(targetDataNode)) {
-1 return String.format(
-1 "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
-1 + "The RemoveDataNodesProcedure is removing the target DataNode %s. "
-1 + "For further information, please search [pid%d] in log. ",
-1 conflictRemoveDataNodesProcedure.get().getProcId(),
-1 targetDataNode,
-1 conflictRemoveDataNodesProcedure.get().getProcId());
-1 }
-1 }
-1 return null;
-1 }
-1
-1 private String checkRegionOperationDuplication(TConsensusGroupId regionId) {
-1 List<? extends RegionOperationProcedure<?>> otherRegionMemberChangeProcedures =
-1 getRegionOperationProcedures()
-1 .filter(
-1 regionMemberChangeProcedure ->
-1 regionId.equals(regionMemberChangeProcedure.getRegionId()))
-1 .collect(Collectors.toList());
-1 if (!otherRegionMemberChangeProcedures.isEmpty()) {
-1 return String.format(
-1 "%s has some other region operation procedures in progress, their procedure id is: %s",
-1 regionId, otherRegionMemberChangeProcedures);
-1 }
-1 return null;
-1 }
-1
-1 public List<TConsensusGroupId> getRegionOperationConsensusIds() {
-1 return getRegionOperationProcedures()
-1 .map(RegionOperationProcedure::getRegionId)
-1 .distinct()
-1 .collect(Collectors.toList());
-1 }
-1
-1 private Stream<RegionOperationProcedure<?>> getRegionOperationProcedures() {
-1 return getExecutor().getProcedures().values().stream()
-1 .filter(procedure -> !procedure.isFinished())
-1 .filter(procedure -> procedure instanceof RegionOperationProcedure)
-1 .map(procedure -> (RegionOperationProcedure<?>) procedure);
-1 }
-1
-1 private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) {
-1 String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId);
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager;
+
+import org.apache.iotdb.common.rpc.thrift.Model;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PathDeserializeUtil;
+import org.apache.iotdb.commons.path.PathPatternTree;
+import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
+import org.apache.iotdb.commons.schema.table.TsTable;
+import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
+import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
+import org.apache.iotdb.commons.service.metric.MetricService;
+import org.apache.iotdb.commons.trigger.TriggerInformation;
+import org.apache.iotdb.commons.utils.StatusUtils;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
+import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
+import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.persistence.ProcedureInfo;
+import org.apache.iotdb.confignode.procedure.PartitionTableAutoCleaner;
+import org.apache.iotdb.confignode.procedure.Procedure;
+import org.apache.iotdb.confignode.procedure.ProcedureExecutor;
+import org.apache.iotdb.confignode.procedure.ProcedureMetrics;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler;
+import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler;
+import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
+import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
+import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
+import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
+import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure;
+import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
+import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleLeaderChangeProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeMetaSyncProcedure;
+import org.apache.iotdb.confignode.procedure.impl.pipe.task.AlterPipeProcedureV2;
+import org.apache.iotdb.confignode.procedure.impl.pipe.task.CreatePipeProcedureV2;
+import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2;
+import org.apache.iotdb.confignode.procedure.impl.pipe.task.StartPipeProcedureV2;
+import org.apache.iotdb.confignode.procedure.impl.pipe.task.StopPipeProcedureV2;
+import org.apache.iotdb.confignode.procedure.impl.region.AddRegionPeerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure;
+import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure;
+import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
+import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan;
+import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure;
+import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.AddViewColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.CreateTableViewProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewColumnProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewProcedure;
+import org.apache.iotdb.confignode.procedure.impl.schema.table.view.SetViewPropertiesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.CreateSubscriptionProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.DropSubscriptionProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.topic.CreateTopicProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.topic.DropTopicProcedure;
+import org.apache.iotdb.confignode.procedure.impl.subscription.topic.runtime.TopicMetaSyncProcedure;
+import org.apache.iotdb.confignode.procedure.impl.sync.AuthOperationProcedure;
+import org.apache.iotdb.confignode.procedure.impl.testonly.AddNeverFinishSubProcedureProcedure;
+import org.apache.iotdb.confignode.procedure.impl.testonly.CreateManyDatabasesProcedure;
+import org.apache.iotdb.confignode.procedure.impl.trigger.CreateTriggerProcedure;
+import org.apache.iotdb.confignode.procedure.impl.trigger.DropTriggerProcedure;
+import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
+import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore;
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+import org.apache.iotdb.confignode.procedure.store.ProcedureFactory;
+import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
+import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TCreateTopicReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
+import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
+import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
+import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
+import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq;
+import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq;
+import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.db.exception.BatchProcessException;
+import org.apache.iotdb.db.schemaengine.template.Template;
+import org.apache.iotdb.db.utils.constant.SqlConstant;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.ratis.util.AutoCloseableLock;
+import org.apache.tsfile.utils.Binary;
+import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.BiFunction;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class ProcedureManager {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureManager.class);
+
+ private static final ConfigNodeConfig CONFIG_NODE_CONFIG =
+ ConfigNodeDescriptor.getInstance().getConf();
+ private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+
+ public static final long PROCEDURE_WAIT_TIME_OUT = COMMON_CONFIG.getDnConnectionTimeoutInMS();
+ public static final int PROCEDURE_WAIT_RETRY_TIMEOUT = 10;
+ private static final String PROCEDURE_TIMEOUT_MESSAGE =
+ "Timed out to wait for procedure return. The procedure is still running.";
+
+ private final ConfigManager configManager;
+ private ProcedureExecutor<ConfigNodeProcedureEnv> executor;
+ private ProcedureScheduler scheduler;
+ private IProcedureStore store;
+ private ConfigNodeProcedureEnv env;
+
+ private final long planSizeLimit;
+ private ProcedureMetrics procedureMetrics;
+
+ private final PartitionTableAutoCleaner partitionTableCleaner;
+
+ private final ReentrantLock tableLock = new ReentrantLock();
+
+ public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo) {
+ this.configManager = configManager;
+ this.scheduler = new SimpleProcedureScheduler();
+ this.store = new ConfigProcedureStore(configManager, procedureInfo);
+ this.env = new ConfigNodeProcedureEnv(configManager, scheduler);
+ this.executor = new ProcedureExecutor<>(env, store, scheduler);
+ this.planSizeLimit =
+ ConfigNodeDescriptor.getInstance()
+ .getConf()
+ .getConfigNodeRatisConsensusLogAppenderBufferSize()
+ - IoTDBConstant.RAFT_LOG_BASIC_SIZE;
+ this.procedureMetrics = new ProcedureMetrics(this);
+ this.partitionTableCleaner = new PartitionTableAutoCleaner<>(configManager);
+ }
+
+ public void startExecutor() {
+ if (!executor.isRunning()) {
+ executor.init(CONFIG_NODE_CONFIG.getProcedureCoreWorkerThreadsCount());
+ executor.startWorkers();
+ executor.startCompletedCleaner(
+ CONFIG_NODE_CONFIG.getProcedureCompletedCleanInterval(),
+ CONFIG_NODE_CONFIG.getProcedureCompletedEvictTTL());
+ executor.addInternalProcedure(partitionTableCleaner);
+ store.start();
+ LOGGER.info("ProcedureManager is started successfully.");
+ }
+ }
+
+ public void stopExecutor() {
+ if (executor.isRunning()) {
+ executor.stop();
+ if (!executor.isRunning()) {
+ executor.join();
+ store.stop();
+ LOGGER.info("ProcedureManager is stopped successfully.");
+ }
+ executor.removeInternalProcedure(partitionTableCleaner);
+ }
+ }
+
+ @TestOnly
+ public TSStatus createManyDatabases() {
+ this.executor.submitProcedure(new CreateManyDatabasesProcedure());
+ return StatusUtils.OK;
+ }
+
+ @TestOnly
+ public TSStatus testSubProcedure() {
+ this.executor.submitProcedure(new AddNeverFinishSubProcedureProcedure());
+ return StatusUtils.OK;
+ }
+
+ public TSStatus deleteDatabases(
+ final List<TDatabaseSchema> deleteSgSchemaList, final boolean isGeneratedByPipe) {
+ final List<DeleteDatabaseProcedure> procedures = new ArrayList<>();
+ final long startCheckTimeForProcedures = System.currentTimeMillis();
+ for (final TDatabaseSchema databaseSchema : deleteSgSchemaList) {
+ final String database = databaseSchema.getName();
+ boolean hasOverlappedTask = false;
+ synchronized (this) {
+ while (executor.isRunning()
+ && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) {
+ final Pair<Long, Boolean> procedureIdDuplicatePair =
+ checkDuplicateTableTask(
+ database, null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE);
+ hasOverlappedTask = procedureIdDuplicatePair.getRight();
+
+ if (Boolean.FALSE.equals(procedureIdDuplicatePair.getRight())) {
+ DeleteDatabaseProcedure procedure =
+ new DeleteDatabaseProcedure(databaseSchema, isGeneratedByPipe);
+ this.executor.submitProcedure(procedure);
+ procedures.add(procedure);
+ break;
+ }
+ try {
+ wait(PROCEDURE_WAIT_RETRY_TIMEOUT);
+ } catch (final InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ String.format(
+ "Some other task is operating table under the database %s, please retry after the procedure finishes.",
+ database));
+ }
+ }
+ }
+ List<TSStatus> results = new ArrayList<>(procedures.size());
+ procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure)));
+ // Clear the previously deleted regions
+ final PartitionManager partitionManager = getConfigManager().getPartitionManager();
+ partitionManager.getRegionMaintainer().submit(partitionManager::maintainRegionReplicas);
+ if (results.stream()
+ .allMatch(result -> result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode())) {
+ return StatusUtils.OK;
+ } else {
+ return RpcUtils.getStatus(results);
+ }
+ }
+
+ public TSStatus deleteTimeSeries(
+ String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) {
+ DeleteTimeSeriesProcedure procedure = null;
+ synchronized (this) {
+ boolean hasOverlappedTask = false;
+ ProcedureType type;
+ DeleteTimeSeriesProcedure deleteTimeSeriesProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.DELETE_TIMESERIES_PROCEDURE)) {
+ continue;
+ }
+ deleteTimeSeriesProcedure = ((DeleteTimeSeriesProcedure) runningProcedure);
+ if (queryId.equals(deleteTimeSeriesProcedure.getQueryId())) {
+ procedure = deleteTimeSeriesProcedure;
+ break;
+ }
+ if (patternTree.isOverlapWith(deleteTimeSeriesProcedure.getPatternTree())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is deleting some target timeseries.");
+ }
+ procedure = new DeleteTimeSeriesProcedure(queryId, patternTree, isGeneratedByPipe);
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ public TSStatus deleteLogicalView(TDeleteLogicalViewReq req) {
+ String queryId = req.getQueryId();
+ PathPatternTree patternTree =
+ PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
+ DeleteLogicalViewProcedure procedure = null;
+ synchronized (this) {
+ boolean hasOverlappedTask = false;
+ ProcedureType type;
+ DeleteLogicalViewProcedure deleteLogicalViewProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.DELETE_LOGICAL_VIEW_PROCEDURE)) {
+ continue;
+ }
+ deleteLogicalViewProcedure = ((DeleteLogicalViewProcedure) runningProcedure);
+ if (queryId.equals(deleteLogicalViewProcedure.getQueryId())) {
+ procedure = deleteLogicalViewProcedure;
+ break;
+ }
+ if (patternTree.isOverlapWith(deleteLogicalViewProcedure.getPatternTree())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is deleting some target views.");
+ }
+ procedure =
+ new DeleteLogicalViewProcedure(
+ queryId, patternTree, req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ public TSStatus alterLogicalView(final TAlterLogicalViewReq req) {
+ final String queryId = req.getQueryId();
+ final ByteBuffer byteBuffer = ByteBuffer.wrap(req.getViewBinary());
+ final Map<PartialPath, ViewExpression> viewPathToSourceMap = new HashMap<>();
+ final int size = byteBuffer.getInt();
+ PartialPath path;
+ ViewExpression viewExpression;
+ for (int i = 0; i < size; i++) {
+ path = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer);
+ viewExpression = ViewExpression.deserialize(byteBuffer);
+ viewPathToSourceMap.put(path, viewExpression);
+ }
+
+ AlterLogicalViewProcedure procedure = null;
+ synchronized (this) {
+ ProcedureType type;
+ AlterLogicalViewProcedure alterLogicalViewProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.ALTER_LOGICAL_VIEW_PROCEDURE)) {
+ continue;
+ }
+ alterLogicalViewProcedure = ((AlterLogicalViewProcedure) runningProcedure);
+ if (queryId.equals(alterLogicalViewProcedure.getQueryId())) {
+ procedure = alterLogicalViewProcedure;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ procedure =
+ new AlterLogicalViewProcedure(
+ queryId,
+ viewPathToSourceMap,
+ req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ public TSStatus setSchemaTemplate(
+ String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) {
+ SetTemplateProcedure procedure = null;
+ synchronized (this) {
+ boolean hasOverlappedTask = false;
+ ProcedureType type;
+ SetTemplateProcedure setTemplateProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
+ continue;
+ }
+ setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
+ if (queryId.equals(setTemplateProcedure.getQueryId())) {
+ procedure = setTemplateProcedure;
+ break;
+ }
+ if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is setting template on target path.");
+ }
+ procedure =
+ new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe);
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ public TSStatus deactivateTemplate(
+ String queryId, Map<PartialPath, List<Template>> templateSetInfo, boolean isGeneratedByPipe) {
+ DeactivateTemplateProcedure procedure = null;
+ synchronized (this) {
+ boolean hasOverlappedTask = false;
+ ProcedureType type;
+ DeactivateTemplateProcedure deactivateTemplateProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.DEACTIVATE_TEMPLATE_PROCEDURE)) {
+ continue;
+ }
+ deactivateTemplateProcedure = (DeactivateTemplateProcedure) runningProcedure;
+ if (queryId.equals(deactivateTemplateProcedure.getQueryId())) {
+ procedure = deactivateTemplateProcedure;
+ break;
+ }
+ for (PartialPath pattern : templateSetInfo.keySet()) {
+ for (PartialPath existingPattern :
+ deactivateTemplateProcedure.getTemplateSetInfo().keySet()) {
+ if (pattern.overlapWith(existingPattern)) {
+ hasOverlappedTask = true;
+ break;
+ }
+ }
+ if (hasOverlappedTask) {
+ break;
+ }
+ }
+ if (hasOverlappedTask) {
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is deactivating some target template from target path.");
+ }
+ procedure = new DeactivateTemplateProcedure(queryId, templateSetInfo, isGeneratedByPipe);
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ public TSStatus unsetSchemaTemplate(
+ String queryId, Template template, PartialPath path, boolean isGeneratedByPipe) {
+ UnsetTemplateProcedure procedure = null;
+ synchronized (this) {
+ boolean hasOverlappedTask = false;
+ ProcedureType type;
+ UnsetTemplateProcedure unsetTemplateProcedure;
+ for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+ type = ProcedureFactory.getProcedureType(runningProcedure);
+ if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
+ continue;
+ }
+ unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
+ if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
+ procedure = unsetTemplateProcedure;
+ break;
+ }
+ if (template.getId() == unsetTemplateProcedure.getTemplateId()
+ && path.equals(unsetTemplateProcedure.getPath())) {
+ hasOverlappedTask = true;
+ break;
+ }
+ }
+
+ if (procedure == null) {
+ if (hasOverlappedTask) {
+ return RpcUtils.getStatus(
+ TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+ "Some other task is unsetting target template from target path "
+ + path.getFullPath());
+ }
+ procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe);
+ this.executor.submitProcedure(procedure);
+ }
+ }
+ return waitingProcedureFinished(procedure);
+ }
+
+ /**
+ * Generate an {@link AddConfigNodeProcedure}, and serially execute all the {@link
+ * AddConfigNodeProcedure}s.
+ */
+ public void addConfigNode(TConfigNodeRegisterReq req) {
+ final AddConfigNodeProcedure addConfigNodeProcedure =
+ new AddConfigNodeProcedure(req.getConfigNodeLocation(), req.getVersionInfo());
+ this.executor.submitProcedure(addConfigNodeProcedure);
+ }
+
+ /**
+ * Generate a {@link RemoveConfigNodeProcedure}, and serially execute all the {@link
+ * RemoveConfigNodeProcedure}s.
+ */
+ public void removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) {
+ final RemoveConfigNodeProcedure removeConfigNodeProcedure =
+ new RemoveConfigNodeProcedure(removeConfigNodePlan.getConfigNodeLocation());
+ this.executor.submitProcedure(removeConfigNodeProcedure);
+ LOGGER.info("Submit RemoveConfigNodeProcedure successfully: {}", removeConfigNodePlan);
+ }
+
+ /**
+ * Generate {@link RemoveDataNodesProcedure}s, and serially execute all the {@link
+ * RemoveDataNodesProcedure}s.
+ */
+ public boolean removeDataNode(RemoveDataNodePlan removeDataNodePlan) {
+ Map<Integer, NodeStatus> nodeStatusMap = new HashMap<>();
+ removeDataNodePlan
+ .getDataNodeLocations()
+ .forEach(
+ datanode ->
+ nodeStatusMap.put(
+ datanode.getDataNodeId(),
+ configManager.getLoadManager().getNodeStatus(datanode.getDataNodeId())));
+ this.executor.submitProcedure(
+ new RemoveDataNodesProcedure(removeDataNodePlan.getDataNodeLocations(), nodeStatusMap));
+ LOGGER.info(
+ "Submit RemoveDataNodesProcedure successfully, {}",
+ removeDataNodePlan.getDataNodeLocations());
+ return true;
+ }
+
+ public boolean removeAINode(RemoveAINodePlan removeAINodePlan) {
+ this.executor.submitProcedure(new RemoveAINodeProcedure(removeAINodePlan.getAINodeLocation()));
+ LOGGER.info(
+ "Submit RemoveAINodeProcedure successfully, {}", removeAINodePlan.getAINodeLocation());
+ return true;
+ }
+
+ public TSStatus checkRemoveDataNodes(List<TDataNodeLocation> dataNodeLocations) {
+ // 1. Only one RemoveDataNodesProcedure is allowed in the cluster
+ Optional<Procedure<ConfigNodeProcedureEnv>> anotherRemoveProcedure =
+ getExecutor().getProcedures().values().stream()
+ .filter(
+ procedure -> {
+ if (procedure instanceof RemoveDataNodesProcedure) {
+ return !procedure.isFinished();
+ }
+ return false;
+ })
+ .findAny();
+
+ String failMessage = null;
+ if (anotherRemoveProcedure.isPresent()) {
+ List<TDataNodeLocation> anotherRemoveDataNodes =
+ ((RemoveDataNodesProcedure) anotherRemoveProcedure.get()).getRemovedDataNodes();
+ failMessage =
+ String.format(
+ "Submit RemoveDataNodesProcedure failed, "
+ + "because another RemoveDataNodesProcedure %s is already in processing. "
+ + "IoTDB is able to have at most 1 RemoveDataNodesProcedure at the same time. "
+ + "For further information, please search [pid%d] in log. ",
+ anotherRemoveDataNodes, anotherRemoveProcedure.get().getProcId());
+ }
+
+ // 2. Check if the RemoveDataNodesProcedure conflicts with the RegionMigrateProcedure
+ Set<TConsensusGroupId> removedDataNodesRegionSet =
+ getEnv().getRemoveDataNodeHandler().getRemovedDataNodesRegionSet(dataNodeLocations);
+ Optional<Procedure<ConfigNodeProcedureEnv>> conflictRegionMigrateProcedure =
+ getExecutor().getProcedures().values().stream()
+ .filter(
+ procedure -> {
+ if (procedure instanceof RegionMigrateProcedure) {
+ RegionMigrateProcedure regionMigrateProcedure =
+ (RegionMigrateProcedure) procedure;
+ if (regionMigrateProcedure.isFinished()) {
+ return false;
+ }
+ return removedDataNodesRegionSet.contains(regionMigrateProcedure.getRegionId())
+ || dataNodeLocations.contains(regionMigrateProcedure.getDestDataNode());
+ }
+ return false;
+ })
+ .findAny();
+ if (conflictRegionMigrateProcedure.isPresent()) {
+ failMessage =
+ String.format(
+ "Submit RemoveDataNodesProcedure failed, "
+ + "because another RegionMigrateProcedure %s is already in processing which conflicts with this RemoveDataNodesProcedure. "
+ + "The RegionMigrateProcedure is migrating the region %s to the DataNode %s. "
+ + "For further information, please search [pid%d] in log. ",
+ conflictRegionMigrateProcedure.get().getProcId(),
+ ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getRegionId(),
+ ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getDestDataNode(),
+ conflictRegionMigrateProcedure.get().getProcId());
+ }
+ // 3. Check if the RegionMigrateProcedure generated by RemoveDataNodesProcedure conflicts with
+ // each other
+ List<RegionMigrationPlan> regionMigrationPlans =
+ getEnv().getRemoveDataNodeHandler().getRegionMigrationPlans(dataNodeLocations);
+ removedDataNodesRegionSet.clear();
+ for (RegionMigrationPlan regionMigrationPlan : regionMigrationPlans) {
+ if (removedDataNodesRegionSet.contains(regionMigrationPlan.getRegionId())) {
+ failMessage =
+ String.format(
+ "Submit RemoveDataNodesProcedure failed, "
+ + "because the RegionMigrateProcedure generated by this RemoveDataNodesProcedure conflicts with each other. "
+ + "Only one replica of the same consensus group is allowed to be migrated at the same time."
+ + "The conflict region id is %s . ",
+ regionMigrationPlan.getRegionId());
+ break;
+ }
+ removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId());
+ }
+
+ // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are
+ // not the remove DataNodes
+
+ for (TDataNodeLocation removeDataNode : dataNodeLocations) {
+ Set<TDataNodeLocation> relatedDataNodes =
+ getEnv().getRemoveDataNodeHandler().getRelatedDataNodeLocations(removeDataNode);
+ relatedDataNodes.remove(removeDataNode);
+
+ for (TDataNodeLocation relatedDataNode : relatedDataNodes) {
+ NodeStatus nodeStatus =
+ getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId());
+ if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) {
+ failMessage =
+ String.format(
+ "Submit RemoveDataNodesProcedure failed, "
+ + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, "
+ + "the remove operation cannot be performed for security reasons. "
+ + "Please check the status of the node %s and ensure it is running.",
+ relatedDataNode.getDataNodeId());
+ }
+ }
+ }
+
+ if (failMessage != null) {
+ LOGGER.warn(failMessage);
+ TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_DATANODE_ERROR.getStatusCode());
+ failStatus.setMessage(failMessage);
+ return failStatus;
+ }
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ // region region operation related check
+
+ /**
+ * Checks whether region migration is allowed.
+ *
+ * @param migrateRegionReq the migration request details
+ * @param regionGroupId the ID of the consensus group for the region
+ * @param originalDataNode the original DataNode location from which the region is being migrated
+ * @param destDataNode the destination DataNode location to which the region is being migrated
+ * @param coordinatorForAddPeer the DataNode location acting as the coordinator for adding a peer
+ * @return the status of the migration request (TSStatus)
+ */
+ private TSStatus checkMigrateRegion(
+ TMigrateRegionReq migrateRegionReq,
+ TConsensusGroupId regionGroupId,
+ TDataNodeLocation originalDataNode,
+ TDataNodeLocation destDataNode,
+ TDataNodeLocation coordinatorForAddPeer) {
+ String failMessage;
+ if ((failMessage =
+ regionOperationCommonCheck(
+ regionGroupId,
+ destDataNode,
+ Arrays.asList(
+ new Pair<>("Original DataNode", originalDataNode),
+ new Pair<>("Destination DataNode", destDataNode),
+ new Pair<>("Coordinator for add peer", coordinatorForAddPeer)),
+ migrateRegionReq.getModel()))
+ != null) {
+ // do nothing
+ } else if (configManager
+ .getPartitionManager()
+ .getAllReplicaSets(originalDataNode.getDataNodeId())
+ .stream()
+ .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
+ failMessage =
+ String.format(
+ "Submit RegionMigrateProcedure failed, because the original DataNode %s doesn't contain Region %s",
+ migrateRegionReq.getFromId(), migrateRegionReq.getRegionId());
+ } else if (configManager
+ .getPartitionManager()
+ .getAllReplicaSets(destDataNode.getDataNodeId())
+ .stream()
+ .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
+ failMessage =
+ String.format(
+ "Submit RegionMigrateProcedure failed, because the target DataNode %s already contains Region %s",
+ migrateRegionReq.getToId(), migrateRegionReq.getRegionId());
+ }
+
+ if (failMessage != null) {
+ LOGGER.warn(failMessage);
+ TSStatus failStatus = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ failStatus.setMessage(failMessage);
+ return failStatus;
+ }
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ private TSStatus checkReconstructRegion(
+ TReconstructRegionReq req,
+ TConsensusGroupId regionId,
+ TDataNodeLocation targetDataNode,
+ TDataNodeLocation coordinator) {
+ String failMessage =
+ regionOperationCommonCheck(
+ regionId,
+ targetDataNode,
+ Arrays.asList(
+ new Pair<>("Target DataNode", targetDataNode),
+ new Pair<>("Coordinator", coordinator)),
+ req.getModel());
+
+ ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
+ if (configManager
+ .getPartitionManager()
+ .getAllReplicaSetsMap(regionId.getType())
+ .get(regionId)
+ .getDataNodeLocationsSize()
+ == 1) {
+ failMessage = String.format("%s only has 1 replica, it cannot be reconstructed", regionId);
+ } else if (configManager
+ .getPartitionManager()
+ .getAllReplicaSets(targetDataNode.getDataNodeId())
+ .stream()
+ .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+ failMessage =
+ String.format(
+ "Submit ReconstructRegionProcedure failed, because the target DataNode %s doesn't contain Region %s",
+ req.getDataNodeId(), regionId);
+ }
+
+ if (failMessage != null) {
+ LOGGER.warn(failMessage);
+ TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
+ failStatus.setMessage(failMessage);
+ return failStatus;
+ }
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ private TSStatus checkExtendRegion(
+ TExtendRegionReq req,
+ TConsensusGroupId regionId,
+ TDataNodeLocation targetDataNode,
+ TDataNodeLocation coordinator) {
+ String failMessage =
+ regionOperationCommonCheck(
+ regionId,
+ targetDataNode,
+ Arrays.asList(
+ new Pair<>("Target DataNode", targetDataNode),
+ new Pair<>("Coordinator", coordinator)),
+ req.getModel());
+ if (configManager
+ .getPartitionManager()
+ .getAllReplicaSets(targetDataNode.getDataNodeId())
+ .stream()
+ .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+ failMessage =
+ String.format(
+ "Target DataNode %s already contains region %s",
+ targetDataNode.getDataNodeId(), regionId);
+ }
+
+ if (failMessage != null) {
+ LOGGER.warn(failMessage);
+ TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
+ failStatus.setMessage(failMessage);
+ return failStatus;
+ }
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ private TSStatus checkRemoveRegion(
+ TRemoveRegionReq req,
+ TConsensusGroupId regionId,
+ @Nullable TDataNodeLocation targetDataNode,
+ TDataNodeLocation coordinator) {
+ String failMessage =
+ regionOperationCommonCheck(
+ regionId,
+ targetDataNode,
+ Arrays.asList(new Pair<>("Coordinator", coordinator)),
+ req.getModel());
+
+ if (configManager
+ .getPartitionManager()
+ .getAllReplicaSetsMap(regionId.getType())
+ .get(regionId)
+ .getDataNodeLocationsSize()
+ == 1) {
+ failMessage = String.format("%s only has 1 replica, it cannot be removed", regionId);
+ } else if (targetDataNode != null
+ && configManager
+ .getPartitionManager()
+ .getAllReplicaSets(targetDataNode.getDataNodeId())
+ .stream()
+ .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+ failMessage =
+ String.format(
+ "Target DataNode %s doesn't contain Region %s", req.getDataNodeId(), regionId);
+ }
+
+ if (failMessage != null) {
+ LOGGER.warn(failMessage);
+ TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_REGION_PEER_ERROR.getStatusCode());
+ failStatus.setMessage(failMessage);
+ return failStatus;
+ }
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ /**
+ * The common checks of all region operations, include migration, reconstruction, extension,
+ * removing
+ *
+ * @param regionId region group id, also called consensus group id
+ * @param targetDataNode DataNode should in Running status
+ * @param relatedDataNodes Pair<Identity, Node Location>
+ * @return The reason if check failed, or null if check pass
+ */
+ private String regionOperationCommonCheck(
+ TConsensusGroupId regionId,
+ TDataNodeLocation targetDataNode,
+ List<Pair<String, TDataNodeLocation>> relatedDataNodes,
+ Model model) {
+ String failMessage;
+ ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
+
+ if (TConsensusGroupType.DataRegion == regionId.getType()
+ && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) {
+ failMessage = "SimpleConsensus not supports region operation.";
+ } else if (TConsensusGroupType.SchemaRegion == regionId.getType()
+ && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getSchemaRegionConsensusProtocolClass())) {
+ failMessage = "SimpleConsensus not supports region operation.";
+ } else if ((failMessage = checkRegionOperationDuplication(regionId)) != null) {
+ // need to do nothing more
+ } else if (relatedDataNodes.stream().anyMatch(pair -> pair.getRight() == null)) {
+ Pair<String, TDataNodeLocation> nullPair =
+ relatedDataNodes.stream().filter(pair -> pair.getRight() == null).findAny().get();
+ failMessage = String.format("Cannot find %s", nullPair.getLeft());
+ } else if (targetDataNode != null
+ && !configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running).stream()
+ .map(TDataNodeConfiguration::getLocation)
+ .map(TDataNodeLocation::getDataNodeId)
+ .collect(Collectors.toSet())
+ .contains(targetDataNode.getDataNodeId())) {
+ // Here we only check Running DataNode to implement migration, because removing nodes may not
+ // exist when add peer is performing
+ failMessage =
+ String.format(
+ "Target DataNode %s is not in Running status.", targetDataNode.getDataNodeId());
+ } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode))
+ != null) {
+ // need to do nothing more
+ } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) {
+ // need to do nothing more
+ }
+
+ return failMessage;
+ }
+
+ private String checkRegionOperationWithRemoveDataNode(
+ TConsensusGroupId regionId, TDataNodeLocation targetDataNode) {
+ Optional<Procedure<ConfigNodeProcedureEnv>> conflictRemoveDataNodesProcedure =
+ getExecutor().getProcedures().values().stream()
+ .filter(
+ procedure -> {
+ if (procedure instanceof RemoveDataNodesProcedure) {
+ return !procedure.isFinished();
+ }
+ return false;
+ })
+ .findAny();
+
+ if (conflictRemoveDataNodesProcedure.isPresent()) {
+ RemoveDataNodeHandler removeDataNodeHandler = env.getRemoveDataNodeHandler();
+ List<TDataNodeLocation> removedDataNodes =
+ ((RemoveDataNodesProcedure) conflictRemoveDataNodesProcedure.get()).getRemovedDataNodes();
+ Set<TConsensusGroupId> removedDataNodesRegionSet =
+ removeDataNodeHandler.getRemovedDataNodesRegionSet(removedDataNodes);
+ if (removedDataNodesRegionSet.contains(regionId)) {
+ return String.format(
+ "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
+ + "The RemoveDataNodesProcedure is removing the DataNodes %s which contains the region %s. "
+ + "For further information, please search [pid%d] in log. ",
+ conflictRemoveDataNodesProcedure.get().getProcId(),
+ removedDataNodes,
+ regionId,
+ conflictRemoveDataNodesProcedure.get().getProcId());
+ } else if (removedDataNodes.contains(targetDataNode)) {
+ return String.format(
+ "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
+ + "The RemoveDataNodesProcedure is removing the target DataNode %s. "
+ + "For further information, please search [pid%d] in log. ",
+ conflictRemoveDataNodesProcedure.get().getProcId(),
+ targetDataNode,
+ conflictRemoveDataNodesProcedure.get().getProcId());
+ }
+ }
+ return null;
+ }
+
+ private String checkRegionOperationDuplication(TConsensusGroupId regionId) {
+ List<? extends RegionOperationProcedure<?>> otherRegionMemberChangeProcedures =
+ getRegionOperationProcedures()
+ .filter(
+ regionMemberChangeProcedure ->
+ regionId.equals(regionMemberChangeProcedure.getRegionId()))
+ .collect(Collectors.toList());
+ if (!otherRegionMemberChangeProcedures.isEmpty()) {
+ return String.format(
+ "%s has some other region operation procedures in progress, their procedure id is: %s",
+ regionId, otherRegionMemberChangeProcedures);
+ }
+ return null;
+ }
+
+ public List<TConsensusGroupId> getRegionOperationConsensusIds() {
+ return getRegionOperationProcedures()
+ .map(RegionOperationProcedure::getRegionId)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ private Stream<RegionOperationProcedure<?>> getRegionOperationProcedures() {
+ return getExecutor().getProcedures().values().stream()
+ .filter(procedure -> !procedure.isFinished())
+ .filter(procedure -> procedure instanceof RegionOperationProcedure)
+ .map(procedure -> (RegionOperationProcedure<?>) procedure);
+ }
+
+ private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) {
+ String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId);
boolean isTreeModelDatabase = databaseName.startsWith(SqlConstant.TREE_MODEL_DATABASE_PREFIX);
if (Model.TREE == model && isTreeModelDatabase
|| Model.TABLE == model && !isTreeModelDatabase) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
index b61cdf46535..ba6712a983d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
@@ -1,37 +1,36 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1public class CompletedProcedureContainer<Env> {
-1 private final Procedure<Env> procedure;
-1
-1 public CompletedProcedureContainer(Procedure<Env> procedure) {
-1 this.procedure = procedure;
-1 }
-1
-1 public Procedure<Env> getProcedure() {
-1 return procedure;
-1 }
-1
-1 public boolean isExpired(long now, long evictTtl) {
-1 return (now - procedure.getLastUpdate()) >= evictTtl;
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+public class CompletedProcedureContainer<Env> {
+ private final Procedure<Env> procedure;
+
+ public CompletedProcedureContainer(Procedure<Env> procedure) {
+ this.procedure = procedure;
+ }
+
+ public Procedure<Env> getProcedure() {
+ return procedure;
+ }
+
+ public boolean isExpired(long now, long evictTtl) {
+ return (now - procedure.getLastUpdate()) >= evictTtl;
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
index aea91dbdb22..cea15a1bfb6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
@@ -1,85 +1,84 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.util.Iterator;
-1import java.util.Map;
-1import java.util.concurrent.TimeUnit;
-1
-1/** Internal cleaner that removes the completed procedure results after a TTL. */
-1public class CompletedProcedureRecycler<Env> extends InternalProcedure<Env> {
-1 private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureRecycler.class);
-1 private static final int DEFAULT_BATCH_SIZE = 8;
-1 private final long evictTTL;
-1 private final Map<Long, CompletedProcedureContainer<Env>> completed;
-1 private final IProcedureStore<Env> store;
-1
-1 public CompletedProcedureRecycler(
-1 IProcedureStore<Env> store,
-1 Map<Long, CompletedProcedureContainer<Env>> completedMap,
-1 long cleanTimeInterval,
-1 long evictTTL) {
-1 super(TimeUnit.SECONDS.toMillis(cleanTimeInterval));
-1 this.completed = completedMap;
-1 this.store = store;
-1 this.evictTTL = evictTTL;
-1 }
-1
-1 @Override
-1 protected void periodicExecute(final Env env) {
-1 if (completed.isEmpty()) {
-1 if (LOG.isTraceEnabled()) {
-1 LOG.trace("No completed procedures to cleanup.");
-1 }
-1 return;
-1 }
-1
-1 final long[] batchIds = new long[DEFAULT_BATCH_SIZE];
-1 int batchCount = 0;
-1
-1 final long now = System.currentTimeMillis();
-1 final Iterator<Map.Entry<Long, CompletedProcedureContainer<Env>>> it =
-1 completed.entrySet().iterator();
-1 while (it.hasNext() && store.isRunning()) {
-1 final Map.Entry<Long, CompletedProcedureContainer<Env>> entry = it.next();
-1 final CompletedProcedureContainer<Env> retainer = entry.getValue();
-1 final Procedure<?> proc = retainer.getProcedure();
-1 if (retainer.isExpired(now, evictTTL)) {
-1 // Failed procedures aren't persisted in WAL.
-1 batchIds[batchCount++] = entry.getKey();
-1 if (batchCount == batchIds.length) {
-1 store.delete(batchIds, 0, batchCount);
-1 batchCount = 0;
-1 }
-1 it.remove();
-1 LOG.trace("Evict completed {}", proc);
-1 }
-1 }
-1 if (batchCount > 0) {
-1 store.delete(batchIds, 0, batchCount);
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/** Internal cleaner that removes the completed procedure results after a TTL. */
+public class CompletedProcedureRecycler<Env> extends InternalProcedure<Env> {
+ private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureRecycler.class);
+ private static final int DEFAULT_BATCH_SIZE = 8;
+ private final long evictTTL;
+ private final Map<Long, CompletedProcedureContainer<Env>> completed;
+ private final IProcedureStore<Env> store;
+
+ public CompletedProcedureRecycler(
+ IProcedureStore<Env> store,
+ Map<Long, CompletedProcedureContainer<Env>> completedMap,
+ long cleanTimeInterval,
+ long evictTTL) {
+ super(TimeUnit.SECONDS.toMillis(cleanTimeInterval));
+ this.completed = completedMap;
+ this.store = store;
+ this.evictTTL = evictTTL;
+ }
+
+ @Override
+ protected void periodicExecute(final Env env) {
+ if (completed.isEmpty()) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("No completed procedures to cleanup.");
+ }
+ return;
+ }
+
+ final long[] batchIds = new long[DEFAULT_BATCH_SIZE];
+ int batchCount = 0;
+
+ final long now = System.currentTimeMillis();
+ final Iterator<Map.Entry<Long, CompletedProcedureContainer<Env>>> it =
+ completed.entrySet().iterator();
+ while (it.hasNext() && store.isRunning()) {
+ final Map.Entry<Long, CompletedProcedureContainer<Env>> entry = it.next();
+ final CompletedProcedureContainer<Env> retainer = entry.getValue();
+ final Procedure<?> proc = retainer.getProcedure();
+ if (retainer.isExpired(now, evictTTL)) {
+ // Failed procedures aren't persisted in WAL.
+ batchIds[batchCount++] = entry.getKey();
+ if (batchCount == batchIds.length) {
+ store.delete(batchIds, 0, batchCount);
+ batchCount = 0;
+ }
+ it.remove();
+ LOG.trace("Evict completed {}", proc);
+ }
+ }
+ if (batchCount > 0) {
+ store.delete(batchIds, 0, batchCount);
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
index 4a70aed89d0..f9ff68bde7e 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
@@ -1,50 +1,49 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import java.io.IOException;
-1import java.nio.ByteBuffer;
-1
-1/**
-1 * Internal Procedure, do some periodic job for framework.
-1 *
-1 * @param <Env>
-1 */
-1public abstract class InternalProcedure<Env> extends Procedure<Env> {
-1 protected InternalProcedure(long toMillis) {
-1 setTimeout(toMillis);
-1 }
-1
-1 protected abstract void periodicExecute(final Env env);
-1
-1 @Override
-1 protected Procedure<Env>[] execute(Env env) throws InterruptedException {
-1 throw new UnsupportedOperationException();
-1 }
-1
-1 @Override
-1 protected void rollback(Env env) throws IOException, InterruptedException {
-1 throw new UnsupportedOperationException();
-1 }
-1
-1 @Override
-1 public void deserialize(ByteBuffer byteBuffer) {}
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * Internal Procedure, do some periodic job for framework.
+ *
+ * @param <Env>
+ */
+public abstract class InternalProcedure<Env> extends Procedure<Env> {
+ protected InternalProcedure(long toMillis) {
+ setTimeout(toMillis);
+ }
+
+ protected abstract void periodicExecute(final Env env);
+
+ @Override
+ protected Procedure<Env>[] execute(Env env) throws InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected void rollback(Env env) throws IOException, InterruptedException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void deserialize(ByteBuffer byteBuffer) {}
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index 4d2d8a23256..2ab15283207 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -1,113 +1,112 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
-1import org.apache.iotdb.commons.conf.CommonConfig;
-1import org.apache.iotdb.commons.conf.CommonDescriptor;
-1import org.apache.iotdb.commons.utils.PathUtils;
-1import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
-1import org.apache.iotdb.confignode.manager.ConfigManager;
-1import org.apache.iotdb.consensus.exception.ConsensusException;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.util.List;
-1import java.util.Map;
-1import java.util.TreeMap;
-1
-1import static org.apache.iotdb.confignode.manager.partition.PartitionManager.CONSENSUS_WRITE_ERROR;
-1
-1/** A cleaner that automatically deletes the expired mapping within the partition table. */
-1public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
-1
-1 private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
-1
-1 private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
-1
-1 private static final String timestampPrecision =
-1 CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-1
-1 private final ConfigManager configManager;
-1
-1 public PartitionTableAutoCleaner(ConfigManager configManager) {
-1 super(COMMON_CONFIG.getTTLCheckInterval());
-1 this.configManager = configManager;
-1 LOGGER.info(
-1 "[PartitionTableCleaner] The PartitionTableAutoCleaner is started with cycle={}ms",
-1 COMMON_CONFIG.getTTLCheckInterval());
-1 }
-1
-1 @Override
-1 protected void periodicExecute(Env env) {
-1 List<String> databases = configManager.getClusterSchemaManager().getDatabaseNames(null);
-1 Map<String, Long> databaseTTLMap = new TreeMap<>();
-1 for (String database : databases) {
-1 long databaseTTL =
-1 PathUtils.isTableModelDatabase(database)
-1 ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database)
-1 : configManager.getTTLManager().getDatabaseMaxTTL(database);
-1 databaseTTLMap.put(database, databaseTTL);
-1 }
-1 LOGGER.info(
-1 "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}",
-1 databaseTTLMap);
-1 for (String database : databases) {
-1 long databaseTTL = databaseTTLMap.get(database);
-1 if (!configManager.getPartitionManager().isDatabaseExist(database)
-1 || databaseTTL < 0
-1 || databaseTTL == Long.MAX_VALUE) {
-1 // Remove the entry if the database or the TTL does not exist
-1 databaseTTLMap.remove(database);
-1 }
-1 }
-1 if (!databaseTTLMap.isEmpty()) {
-1 LOGGER.info(
-1 "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
-1 databaseTTLMap);
-1 // Only clean the partition table when necessary
-1 TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
-1 try {
-1 configManager
-1 .getConsensusManager()
-1 .write(new AutoCleanPartitionTablePlan(databaseTTLMap, currentTimePartitionSlot));
-1 } catch (ConsensusException e) {
-1 LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
-1 }
-1 }
-1 }
-1
-1 /**
-1 * @return The time partition slot corresponding to current timestamp. Note that we do not shift
-1 * the start time to the correct starting point, since this interface only constructs a time
-1 * reference position for the partition table cleaner.
-1 */
-1 private static TTimePartitionSlot getCurrentTimePartitionSlot() {
-1 if ("ms".equals(timestampPrecision)) {
-1 return new TTimePartitionSlot(System.currentTimeMillis());
-1 } else if ("us".equals(timestampPrecision)) {
-1 return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
-1 } else {
-1 return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.commons.utils.PathUtils;
+import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static org.apache.iotdb.confignode.manager.partition.PartitionManager.CONSENSUS_WRITE_ERROR;
+
+/** A cleaner that automatically deletes the expired mapping within the partition table. */
+public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
+
+ private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+
+ private static final String timestampPrecision =
+ CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+
+ private final ConfigManager configManager;
+
+ public PartitionTableAutoCleaner(ConfigManager configManager) {
+ super(COMMON_CONFIG.getTTLCheckInterval());
+ this.configManager = configManager;
+ LOGGER.info(
+ "[PartitionTableCleaner] The PartitionTableAutoCleaner is started with cycle={}ms",
+ COMMON_CONFIG.getTTLCheckInterval());
+ }
+
+ @Override
+ protected void periodicExecute(Env env) {
+ List<String> databases = configManager.getClusterSchemaManager().getDatabaseNames(null);
+ Map<String, Long> databaseTTLMap = new TreeMap<>();
+ for (String database : databases) {
+ long databaseTTL =
+ PathUtils.isTableModelDatabase(database)
+ ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database)
+ : configManager.getTTLManager().getDatabaseMaxTTL(database);
+ databaseTTLMap.put(database, databaseTTL);
+ }
+ LOGGER.info(
+ "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}",
+ databaseTTLMap);
+ for (String database : databases) {
+ long databaseTTL = databaseTTLMap.get(database);
+ if (!configManager.getPartitionManager().isDatabaseExist(database)
+ || databaseTTL < 0
+ || databaseTTL == Long.MAX_VALUE) {
+ // Remove the entry if the database or the TTL does not exist
+ databaseTTLMap.remove(database);
+ }
+ }
+ if (!databaseTTLMap.isEmpty()) {
+ LOGGER.info(
+ "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
+ databaseTTLMap);
+ // Only clean the partition table when necessary
+ TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
+ try {
+ configManager
+ .getConsensusManager()
+ .write(new AutoCleanPartitionTablePlan(databaseTTLMap, currentTimePartitionSlot));
+ } catch (ConsensusException e) {
+ LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
+ }
+ }
+ }
+
+ /**
+ * @return The time partition slot corresponding to current timestamp. Note that we do not shift
+ * the start time to the correct starting point, since this interface only constructs a time
+ * reference position for the partition table cleaner.
+ */
+ private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+ if ("ms".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis());
+ } else if ("us".equals(timestampPrecision)) {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+ } else {
+ return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
index 7eb9eaea987..91af03d3971 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
@@ -1,824 +1,823 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-1import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
-1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.io.DataOutputStream;
-1import java.io.IOException;
-1import java.nio.ByteBuffer;
-1import java.nio.charset.StandardCharsets;
-1import java.util.ArrayList;
-1import java.util.Arrays;
-1import java.util.List;
-1import java.util.concurrent.atomic.AtomicReference;
-1
-1/**
-1 * Abstract class of all procedures.
-1 *
-1 * @param <Env>
-1 */
-1public abstract class Procedure<Env> implements Comparable<Procedure<Env>> {
-1 private static final Logger LOG = LoggerFactory.getLogger(Procedure.class);
-1 public static final long NO_PROC_ID = -1;
-1 public static final long NO_TIMEOUT = -1;
-1
-1 private long parentProcId = NO_PROC_ID;
-1 private long rootProcId = NO_PROC_ID;
-1 private long procId = NO_PROC_ID;
-1 private long submittedTime;
-1
-1 private ProcedureState state = ProcedureState.INITIALIZING;
-1 private int childrenLatch = 0;
-1 private ProcedureException exception;
-1
-1 private volatile long timeout = NO_TIMEOUT;
-1 private volatile long lastUpdate;
-1
-1 private final AtomicReference<byte[]> result = new AtomicReference<>();
-1 private volatile boolean locked = false;
-1 private boolean lockedWhenLoading = false;
-1
-1 private int[] stackIndexes = null;
-1
-1 public final boolean hasLock() {
-1 return locked;
-1 }
-1
-1 // User level code, override it if necessary
-1
-1 /**
-1 * The main code of the procedure. It must be idempotent since execute() may be called multiple
-1 * times in case of machine failure in the middle of the execution.
-1 *
-1 * @param env the environment passed to the ProcedureExecutor
-1 * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the
-1 * procedure is done.
-1 * @throws InterruptedException the procedure will be added back to the queue and retried later.
-1 */
-1 protected abstract Procedure<Env>[] execute(Env env) throws InterruptedException;
-1
-1 /**
-1 * The code to undo what was done by the execute() code. It is called when the procedure or one of
-1 * the sub-procedures failed or an abort was requested. It should cleanup all the resources
-1 * created by the execute() call. The implementation must be idempotent since rollback() may be
-1 * called multiple time in case of machine failure in the middle of the execution.
-1 *
-1 * @param env the environment passed to the ProcedureExecutor
-1 * @throws IOException temporary failure, the rollback will retry later
-1 * @throws InterruptedException the procedure will be added back to the queue and retried later
-1 */
-1 protected abstract void rollback(Env env)
-1 throws IOException, InterruptedException, ProcedureException;
-1
-1 public void serialize(DataOutputStream stream) throws IOException {
-1 // procid
-1 stream.writeLong(this.procId);
-1 // state
-1 stream.writeInt(this.state.ordinal());
-1 // submit time
-1 stream.writeLong(this.submittedTime);
-1 // last updated
-1 stream.writeLong(this.lastUpdate);
-1 // parent id
-1 stream.writeLong(this.parentProcId);
-1 // time out
-1 stream.writeLong(this.timeout);
-1 // stack indexes
-1 if (stackIndexes != null) {
-1 stream.writeInt(stackIndexes.length);
-1 for (int index : stackIndexes) {
-1 stream.writeInt(index);
-1 }
-1 } else {
-1 stream.writeInt(-1);
-1 }
-1
-1 // exceptions
-1 if (hasException()) {
-1 // symbol of exception
-1 stream.write((byte) 1);
-1 // exception's name
-1 String exceptionClassName = exception.getClass().getName();
-1 byte[] exceptionClassNameBytes = exceptionClassName.getBytes(StandardCharsets.UTF_8);
-1 stream.writeInt(exceptionClassNameBytes.length);
-1 stream.write(exceptionClassNameBytes);
-1 // exception's message
-1 String message = this.exception.getMessage();
-1 if (message != null) {
-1 byte[] messageBytes = message.getBytes(StandardCharsets.UTF_8);
-1 stream.writeInt(messageBytes.length);
-1 stream.write(messageBytes);
-1 } else {
-1 stream.writeInt(-1);
-1 }
-1 } else {
-1 // symbol of no exception
-1 stream.write((byte) 0);
-1 }
-1
-1 // result
-1 if (result.get() != null) {
-1 stream.writeInt(result.get().length);
-1 stream.write(result.get());
-1 } else {
-1 stream.writeInt(-1);
-1 }
-1
-1 // Has lock
-1 stream.write(this.hasLock() ? (byte) 1 : (byte) 0);
-1 }
-1
-1 public void deserialize(ByteBuffer byteBuffer) {
-1 // procid
-1 this.setProcId(byteBuffer.getLong());
-1 // state
-1 this.setState(ProcedureState.values()[byteBuffer.getInt()]);
-1 // submit time
-1 this.setSubmittedTime(byteBuffer.getLong());
-1 // last updated
-1 this.setLastUpdate(byteBuffer.getLong());
-1 // parent id
-1 this.setParentProcId(byteBuffer.getLong());
-1 // time out
-1 this.setTimeout(byteBuffer.getLong());
-1 // stack index
-1 int stackIndexesLen = byteBuffer.getInt();
-1 if (stackIndexesLen >= 0) {
-1 List<Integer> indexList = new ArrayList<>(stackIndexesLen);
-1 for (int i = 0; i < stackIndexesLen; i++) {
-1 indexList.add(byteBuffer.getInt());
-1 }
-1 this.setStackIndexes(indexList);
-1 }
-1
-1 // exception
-1 if (byteBuffer.get() == 1) {
-1 deserializeTypeInfoForCompatibility(byteBuffer);
-1 int messageBytesLength = byteBuffer.getInt();
-1 String errMsg = null;
-1 if (messageBytesLength > 0) {
-1 byte[] messageBytes = new byte[messageBytesLength];
-1 byteBuffer.get(messageBytes);
-1 errMsg = new String(messageBytes, StandardCharsets.UTF_8);
-1 }
-1 setFailure(new ProcedureException(errMsg));
-1 }
-1
-1 // result
-1 int resultLen = byteBuffer.getInt();
-1 if (resultLen > 0) {
-1 byte[] resultArr = new byte[resultLen];
-1 byteBuffer.get(resultArr);
-1 }
-1 // has lock
-1 if (byteBuffer.get() == 1) {
-1 this.lockedWhenLoading();
-1 }
-1 }
-1
-1 /**
-1 * Deserialize class Name and load class
-1 *
-1 * @param byteBuffer bytebuffer
-1 * @return Procedure
-1 */
-1 @Deprecated
-1 public static void deserializeTypeInfoForCompatibility(ByteBuffer byteBuffer) {
-1 int classNameBytesLen = byteBuffer.getInt();
-1 byte[] classNameBytes = new byte[classNameBytesLen];
-1 byteBuffer.get(classNameBytes);
-1 }
-1
-1 /**
-1 * Acquire a lock, user should override it if necessary.
-1 *
-1 * @param env environment
-1 * @return state of lock
-1 */
-1 protected ProcedureLockState acquireLock(Env env) {
-1 return ProcedureLockState.LOCK_ACQUIRED;
-1 }
-1
-1 /**
-1 * Release a lock, user should override it if necessary.
-1 *
-1 * @param env env
-1 */
-1 protected void releaseLock(Env env) {
-1 // no op
-1 }
-1
-1 /**
-1 * Used to keep procedure lock even when the procedure is yielded or suspended.
-1 *
-1 * @param env env
-1 * @return true if hold the lock
-1 */
-1 protected boolean holdLock(Env env) {
-1 return false;
-1 }
-1
-1 /**
-1 * To make executor yield between each execution step to give other procedures a chance to run.
-1 *
-1 * @param env environment
-1 * @return return true if yield is allowed.
-1 */
-1 protected boolean isYieldAfterExecution(Env env) {
-1 return false;
-1 }
-1
-1 // -------------------------Internal methods - called by the procedureExecutor------------------
-1 /**
-1 * Internal method called by the ProcedureExecutor that starts the user-level code execute().
-1 *
-1 * @param env execute environment
-1 * @return sub procedures
-1 */
-1 protected Procedure<Env>[] doExecute(Env env) throws InterruptedException {
-1 try {
-1 updateTimestamp();
-1 return execute(env);
-1 } finally {
-1 updateTimestamp();
-1 }
-1 }
-1
-1 /**
-1 * Internal method called by the ProcedureExecutor that starts the user-level code rollback().
-1 *
-1 * @param env execute environment
-1 * @throws IOException ioe
-1 * @throws InterruptedException interrupted exception
-1 */
-1 public void doRollback(Env env) throws IOException, InterruptedException, ProcedureException {
-1 try {
-1 updateTimestamp();
-1 rollback(env);
-1 } finally {
-1 updateTimestamp();
-1 }
-1 }
-1
-1 /**
-1 * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock().
-1 *
-1 * @param env environment
-1 * @param store ProcedureStore
-1 * @return ProcedureLockState
-1 */
-1 public final ProcedureLockState doAcquireLock(Env env, IProcedureStore store) {
-1 if (lockedWhenLoading) {
-1 lockedWhenLoading = false;
-1 locked = true;
-1 return ProcedureLockState.LOCK_ACQUIRED;
-1 }
-1 ProcedureLockState state = acquireLock(env);
-1 if (state == ProcedureLockState.LOCK_ACQUIRED) {
-1 locked = true;
-1 store.update(this);
-1 }
-1 return state;
-1 }
-1
-1 /**
-1 * Presist lock state of the procedure
-1 *
-1 * @param env environment
-1 * @param store ProcedureStore
-1 */
-1 public final void doReleaseLock(Env env, IProcedureStore store) {
-1 locked = false;
-1 if (getState() != ProcedureState.ROLLEDBACK) {
-1 store.update(this);
-1 }
-1 releaseLock(env);
-1 }
-1
-1 public final void restoreLock(Env env) {
-1 if (!lockedWhenLoading) {
-1 LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock", this);
-1 return;
-1 }
-1 if (isFinished()) {
-1 LOG.debug("{} is already bypassed, skip acquiring lock.", this);
-1 return;
-1 }
-1 if (getState() == ProcedureState.WAITING && !holdLock(env)) {
-1 LOG.debug("{} is in WAITING STATE, and holdLock= false , skip acquiring lock.", this);
-1 return;
-1 }
-1 LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this);
-1 acquireLock(env);
-1 }
-1
-1 @Override
-1 public String toString() {
-1 // Return the simple String presentation of the procedure.
-1 return toStringSimpleSB().toString();
-1 }
-1
-1 /**
-1 * Build the StringBuilder for the simple form of procedure string.
-1 *
-1 * @return the StringBuilder
-1 */
-1 protected StringBuilder toStringSimpleSB() {
-1 final StringBuilder sb = new StringBuilder();
-1
-1 sb.append("pid=");
-1 sb.append(getProcId());
-1
-1 if (hasParent()) {
-1 sb.append(", ppid=");
-1 sb.append(getParentProcId());
-1 }
-1
-1 /*
-1 * TODO
-1 * Enable later when this is being used.
-1 * Currently owner not used.
-1 if (hasOwner()) {
-1 sb.append(", owner=");
-1 sb.append(getOwner());
-1 }*/
-1
-1 sb.append(", state="); // pState for Procedure State as opposed to any other kind.
-1 toStringState(sb);
-1
-1 // Only print out locked if actually locked. Most of the time it is not.
-1 if (this.locked) {
-1 sb.append(", locked=").append(locked);
-1 }
-1 if (hasException()) {
-1 sb.append(", exception=" + getException());
-1 }
-1
-1 sb.append("; ");
-1 toStringClassDetails(sb);
-1
-1 return sb;
-1 }
-1
-1 /** Extend the toString() information with more procedure details */
-1 public String toStringDetails() {
-1 final StringBuilder sb = toStringSimpleSB();
-1
-1 sb.append(" submittedTime=");
-1 sb.append(getSubmittedTime());
-1
-1 sb.append(", lastUpdate=");
-1 sb.append(getLastUpdate());
-1
-1 final int[] stackIndices = getStackIndexes();
-1 if (stackIndices != null) {
-1 sb.append("\n");
-1 sb.append("stackIndexes=");
-1 sb.append(Arrays.toString(stackIndices));
-1 }
-1
-1 return sb.toString();
-1 }
-1
-1 /**
-1 * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating
-1 * generic Procedure State with Procedure particulars.
-1 *
-1 * @param builder Append current {@link ProcedureState}
-1 */
-1 protected void toStringState(StringBuilder builder) {
-1 builder.append(getState());
-1 }
-1
-1 /**
-1 * Extend the toString() information with the procedure details e.g. className and parameters
-1 *
-1 * @param builder the string builder to use to append the proc specific information
-1 */
-1 protected void toStringClassDetails(StringBuilder builder) {
-1 builder.append(getClass().getName());
-1 }
-1
-1 // ==========================================================================
-1 // Those fields are unchanged after initialization.
-1 //
-1 // Each procedure will get created from the user or during
-1 // ProcedureExecutor.start() during the load() phase and then submitted
-1 // to the executor. these fields will never be changed after initialization
-1 // ==========================================================================
-1 public long getProcId() {
-1 return procId;
-1 }
-1
-1 public boolean hasParent() {
-1 return parentProcId != NO_PROC_ID;
-1 }
-1
-1 public long getParentProcId() {
-1 return parentProcId;
-1 }
-1
-1 public long getRootProcId() {
-1 return rootProcId;
-1 }
-1
-1 public String getProcType() {
-1 return getClass().getSimpleName();
-1 }
-1
-1 public long getSubmittedTime() {
-1 return submittedTime;
-1 }
-1
-1 /** Called by the ProcedureExecutor to assign the ID to the newly created procedure. */
-1 public void setProcId(long procId) {
-1 this.procId = procId;
-1 }
-1
-1 public void setProcRunnable() {
-1 this.submittedTime = System.currentTimeMillis();
-1 setState(ProcedureState.RUNNABLE);
-1 }
-1
-1 /** Called by the ProcedureExecutor to assign the parent to the newly created procedure. */
-1 protected void setParentProcId(long parentProcId) {
-1 this.parentProcId = parentProcId;
-1 }
-1
-1 protected void setRootProcId(long rootProcId) {
-1 this.rootProcId = rootProcId;
-1 }
-1
-1 /**
-1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
-1 */
-1 protected void setSubmittedTime(long submittedTime) {
-1 this.submittedTime = submittedTime;
-1 }
-1
-1 // ==========================================================================
-1 // runtime state - timeout related
-1 // ==========================================================================
-1 /**
-1 * @param timeout timeout interval in msec
-1 */
-1 protected void setTimeout(long timeout) {
-1 this.timeout = timeout;
-1 }
-1
-1 public boolean hasTimeout() {
-1 return timeout != NO_TIMEOUT;
-1 }
-1
-1 /**
-1 * @return the timeout in msec
-1 */
-1 public long getTimeout() {
-1 return timeout;
-1 }
-1
-1 /**
-1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
-1 */
-1 protected void setLastUpdate(long lastUpdate) {
-1 this.lastUpdate = lastUpdate;
-1 }
-1
-1 /** Called by ProcedureExecutor after each time a procedure step is executed. */
-1 protected void updateTimestamp() {
-1 this.lastUpdate = System.currentTimeMillis();
-1 }
-1
-1 public long getLastUpdate() {
-1 return lastUpdate;
-1 }
-1
-1 /**
-1 * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set
-1 * and the procedure is in the waiting queue.
-1 *
-1 * @return the timestamp of the next timeout.
-1 */
-1 protected long getTimeoutTimestamp() {
-1 return getLastUpdate() + getTimeout();
-1 }
-1
-1 // ==========================================================================
-1 // runtime state
-1 // ==========================================================================
-1 /**
-1 * @return the time elapsed between the last update and the start time of the procedure.
-1 */
-1 public long elapsedTime() {
-1 return getLastUpdate() - getSubmittedTime();
-1 }
-1
-1 /**
-1 * @return the serialized result if any, otherwise null
-1 */
-1 public byte[] getResult() {
-1 return result.get();
-1 }
-1
-1 /**
-1 * The procedure may leave a "result" on completion.
-1 *
-1 * @param result the serialized result that will be passed to the client
-1 */
-1 protected void setResult(byte[] result) {
-1 this.result.set(result);
-1 }
-1
-1 /**
-1 * Will only be called when loading procedures from procedure store, where we need to record
-1 * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)}
-1 * to actually acquire the lock.
-1 */
-1 final void lockedWhenLoading() {
-1 this.lockedWhenLoading = true;
-1 }
-1
-1 /**
-1 * Can only be called when restarting, before the procedure actually being executed, as after we
-1 * actually call the {@link #doAcquireLock(Object, IProcedureStore)} method, we will reset {@link
-1 * #lockedWhenLoading} to false.
-1 *
-1 * <p>Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure
-1 * in front of a queue.
-1 */
-1 public boolean isLockedWhenLoading() {
-1 return lockedWhenLoading;
-1 }
-1
-1 // ==============================================================================================
-1 // Runtime state, updated every operation by the ProcedureExecutor
-1 //
-1 // There is always 1 thread at the time operating on the state of the procedure.
-1 // The ProcedureExecutor may check and set states, or some Procecedure may
-1 // update its own state. but no concurrent updates. we use synchronized here
-1 // just because the procedure can get scheduled on different executor threads on each step.
-1 // ==============================================================================================
-1
-1 /**
-1 * @return true if the procedure is in a RUNNABLE state.
-1 */
-1 public synchronized boolean isRunnable() {
-1 return state == ProcedureState.RUNNABLE;
-1 }
-1
-1 public synchronized boolean isInitializing() {
-1 return state == ProcedureState.INITIALIZING;
-1 }
-1
-1 /**
-1 * @return true if the procedure has failed. It may or may not have rolled back.
-1 */
-1 public synchronized boolean isFailed() {
-1 return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK;
-1 }
-1
-1 /**
-1 * @return true if the procedure is finished successfully.
-1 */
-1 public synchronized boolean isSuccess() {
-1 return state == ProcedureState.SUCCESS && !hasException();
-1 }
-1
-1 /**
-1 * @return true if the procedure is finished. The Procedure may be completed successfully or
-1 * rolledback.
-1 */
-1 public synchronized boolean isFinished() {
-1 return isSuccess() || state == ProcedureState.ROLLEDBACK;
-1 }
-1
-1 /**
-1 * @return true if the procedure is waiting for a child to finish or for an external event.
-1 */
-1 public synchronized boolean isWaiting() {
-1 switch (state) {
-1 case WAITING:
-1 case WAITING_TIMEOUT:
-1 return true;
-1 default:
-1 break;
-1 }
-1 return false;
-1 }
-1
-1 protected synchronized void setState(final ProcedureState state) {
-1 this.state = state;
-1 updateTimestamp();
-1 }
-1
-1 public synchronized ProcedureState getState() {
-1 return state;
-1 }
-1
-1 protected synchronized void setFailure(final String source, final Throwable cause) {
-1 setFailure(new ProcedureException(source, cause));
-1 }
-1
-1 protected synchronized void setFailure(final ProcedureException exception) {
-1 this.exception = exception;
-1 if (!isFinished()) {
-1 setState(ProcedureState.FAILED);
-1 }
-1 }
-1
-1 /**
-1 * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired.
-1 *
-1 * @return true to let the framework handle the timeout as abort, false in case the procedure
-1 * handled the timeout itself.
-1 */
-1 protected synchronized boolean setTimeoutFailure(Env env) {
-1 if (state == ProcedureState.WAITING_TIMEOUT) {
-1 long timeDiff = System.currentTimeMillis() - lastUpdate;
-1 setFailure(
-1 "ProcedureExecutor",
-1 new ProcedureException("Operation timed out after " + timeDiff + " ms."));
-1 return true;
-1 }
-1 return false;
-1 }
-1
-1 public synchronized boolean hasException() {
-1 return exception != null;
-1 }
-1
-1 public synchronized ProcedureException getException() {
-1 return exception;
-1 }
-1
-1 /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
-1 protected synchronized void setChildrenLatch(int numChildren) {
-1 this.childrenLatch = numChildren;
-1 if (LOG.isTraceEnabled()) {
-1 LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString()));
-1 }
-1 }
-1
-1 /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
-1 protected synchronized void incChildrenLatch() {
-1 // TODO: can this be inferred from the stack? I think so...
-1 this.childrenLatch++;
-1 if (LOG.isTraceEnabled()) {
-1 LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString()));
-1 }
-1 }
-1
-1 /** Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. */
-1 private synchronized boolean childrenCountDown() {
-1 assert childrenLatch > 0 : this;
-1 boolean b = --childrenLatch == 0;
-1 if (LOG.isTraceEnabled()) {
-1 LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString()));
-1 }
-1 return b;
-1 }
-1
-1 /**
-1 * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done.
-1 *
-1 * @return True if we were able to move procedure to RUNNABLE state.
-1 */
-1 synchronized boolean tryRunnable() {
-1 // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT
-1 if (getState() == ProcedureState.WAITING && childrenCountDown()) {
-1 setState(ProcedureState.RUNNABLE);
-1 return true;
-1 } else {
-1 return false;
-1 }
-1 }
-1
-1 protected synchronized boolean hasChildren() {
-1 return childrenLatch > 0;
-1 }
-1
-1 protected synchronized int getChildrenLatch() {
-1 return childrenLatch;
-1 }
-1
-1 /**
-1 * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index
-1 * positions.
-1 */
-1 protected synchronized void addStackIndex(final int index) {
-1 if (stackIndexes == null) {
-1 stackIndexes = new int[] {index};
-1 } else {
-1 int count = stackIndexes.length;
-1 stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
-1 stackIndexes[count] = index;
-1 }
-1 }
-1
-1 protected synchronized boolean removeStackIndex() {
-1 if (stackIndexes != null && stackIndexes.length > 1) {
-1 stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
-1 return false;
-1 } else {
-1 stackIndexes = null;
-1 return true;
-1 }
-1 }
-1
-1 /**
-1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
-1 */
-1 protected synchronized void setStackIndexes(final List<Integer> stackIndexes) {
-1 this.stackIndexes = new int[stackIndexes.size()];
-1 for (int i = 0; i < this.stackIndexes.length; ++i) {
-1 this.stackIndexes[i] = stackIndexes.get(i);
-1 }
-1 }
-1
-1 protected synchronized boolean wasExecuted() {
-1 return stackIndexes != null;
-1 }
-1
-1 protected synchronized int[] getStackIndexes() {
-1 return stackIndexes;
-1 }
-1
-1 public void setRootProcedureId(long rootProcedureId) {
-1 this.rootProcId = rootProcedureId;
-1 }
-1
-1 /**
-1 * @param a the first procedure to be compared.
-1 * @param b the second procedure to be compared.
-1 * @return true if the two procedures have the same parent
-1 */
-1 public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) {
-1 return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId());
-1 }
-1
-1 @Override
-1 public int compareTo(Procedure<Env> other) {
-1 return Long.compare(getProcId(), other.getProcId());
-1 }
-1
-1 /**
-1 * This function will be called just when procedure is submitted for execution.
-1 *
-1 * @param env The environment passed to the procedure executor
-1 */
-1 protected void updateMetricsOnSubmit(Env env) {
-1 if (env instanceof ConfigNodeProcedureEnv) {
-1 ((ConfigNodeProcedureEnv) env)
-1 .getConfigManager()
-1 .getProcedureManager()
-1 .getProcedureMetrics()
-1 .updateMetricsOnSubmit(getProcType());
-1 }
-1 }
-1
-1 /**
-1 * This function will be called just after procedure execution is finished. Override this method
-1 * to update metrics at the end of the procedure. The default implementation adds runtime of a
-1 * procedure to a time histogram for successfully completed procedures. Increments failed counter
-1 * for failed procedures.
-1 *
-1 * @param env The environment passed to the procedure executor
-1 * @param runtime Runtime of the procedure in milliseconds
-1 * @param success true if procedure is completed successfully
-1 */
-1 protected void updateMetricsOnFinish(Env env, long runtime, boolean success) {
-1 if (env instanceof ConfigNodeProcedureEnv) {
-1 ((ConfigNodeProcedureEnv) env)
-1 .getConfigManager()
-1 .getProcedureManager()
-1 .getProcedureMetrics()
-1 .updateMetricsOnFinish(getProcType(), runtime, success);
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
+import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Abstract class of all procedures.
+ *
+ * @param <Env>
+ */
+public abstract class Procedure<Env> implements Comparable<Procedure<Env>> {
+ private static final Logger LOG = LoggerFactory.getLogger(Procedure.class);
+ public static final long NO_PROC_ID = -1;
+ public static final long NO_TIMEOUT = -1;
+
+ private long parentProcId = NO_PROC_ID;
+ private long rootProcId = NO_PROC_ID;
+ private long procId = NO_PROC_ID;
+ private long submittedTime;
+
+ private ProcedureState state = ProcedureState.INITIALIZING;
+ private int childrenLatch = 0;
+ private ProcedureException exception;
+
+ private volatile long timeout = NO_TIMEOUT;
+ private volatile long lastUpdate;
+
+ private final AtomicReference<byte[]> result = new AtomicReference<>();
+ private volatile boolean locked = false;
+ private boolean lockedWhenLoading = false;
+
+ private int[] stackIndexes = null;
+
+ public final boolean hasLock() {
+ return locked;
+ }
+
+ // User level code, override it if necessary
+
+ /**
+ * The main code of the procedure. It must be idempotent since execute() may be called multiple
+ * times in case of machine failure in the middle of the execution.
+ *
+ * @param env the environment passed to the ProcedureExecutor
+ * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the
+ * procedure is done.
+ * @throws InterruptedException the procedure will be added back to the queue and retried later.
+ */
+ protected abstract Procedure<Env>[] execute(Env env) throws InterruptedException;
+
+ /**
+ * The code to undo what was done by the execute() code. It is called when the procedure or one of
+ * the sub-procedures failed or an abort was requested. It should cleanup all the resources
+ * created by the execute() call. The implementation must be idempotent since rollback() may be
+ * called multiple time in case of machine failure in the middle of the execution.
+ *
+ * @param env the environment passed to the ProcedureExecutor
+ * @throws IOException temporary failure, the rollback will retry later
+ * @throws InterruptedException the procedure will be added back to the queue and retried later
+ */
+ protected abstract void rollback(Env env)
+ throws IOException, InterruptedException, ProcedureException;
+
+ public void serialize(DataOutputStream stream) throws IOException {
+ // procid
+ stream.writeLong(this.procId);
+ // state
+ stream.writeInt(this.state.ordinal());
+ // submit time
+ stream.writeLong(this.submittedTime);
+ // last updated
+ stream.writeLong(this.lastUpdate);
+ // parent id
+ stream.writeLong(this.parentProcId);
+ // time out
+ stream.writeLong(this.timeout);
+ // stack indexes
+ if (stackIndexes != null) {
+ stream.writeInt(stackIndexes.length);
+ for (int index : stackIndexes) {
+ stream.writeInt(index);
+ }
+ } else {
+ stream.writeInt(-1);
+ }
+
+ // exceptions
+ if (hasException()) {
+ // symbol of exception
+ stream.write((byte) 1);
+ // exception's name
+ String exceptionClassName = exception.getClass().getName();
+ byte[] exceptionClassNameBytes = exceptionClassName.getBytes(StandardCharsets.UTF_8);
+ stream.writeInt(exceptionClassNameBytes.length);
+ stream.write(exceptionClassNameBytes);
+ // exception's message
+ String message = this.exception.getMessage();
+ if (message != null) {
+ byte[] messageBytes = message.getBytes(StandardCharsets.UTF_8);
+ stream.writeInt(messageBytes.length);
+ stream.write(messageBytes);
+ } else {
+ stream.writeInt(-1);
+ }
+ } else {
+ // symbol of no exception
+ stream.write((byte) 0);
+ }
+
+ // result
+ if (result.get() != null) {
+ stream.writeInt(result.get().length);
+ stream.write(result.get());
+ } else {
+ stream.writeInt(-1);
+ }
+
+ // Has lock
+ stream.write(this.hasLock() ? (byte) 1 : (byte) 0);
+ }
+
+ public void deserialize(ByteBuffer byteBuffer) {
+ // procid
+ this.setProcId(byteBuffer.getLong());
+ // state
+ this.setState(ProcedureState.values()[byteBuffer.getInt()]);
+ // submit time
+ this.setSubmittedTime(byteBuffer.getLong());
+ // last updated
+ this.setLastUpdate(byteBuffer.getLong());
+ // parent id
+ this.setParentProcId(byteBuffer.getLong());
+ // time out
+ this.setTimeout(byteBuffer.getLong());
+ // stack index
+ int stackIndexesLen = byteBuffer.getInt();
+ if (stackIndexesLen >= 0) {
+ List<Integer> indexList = new ArrayList<>(stackIndexesLen);
+ for (int i = 0; i < stackIndexesLen; i++) {
+ indexList.add(byteBuffer.getInt());
+ }
+ this.setStackIndexes(indexList);
+ }
+
+ // exception
+ if (byteBuffer.get() == 1) {
+ deserializeTypeInfoForCompatibility(byteBuffer);
+ int messageBytesLength = byteBuffer.getInt();
+ String errMsg = null;
+ if (messageBytesLength > 0) {
+ byte[] messageBytes = new byte[messageBytesLength];
+ byteBuffer.get(messageBytes);
+ errMsg = new String(messageBytes, StandardCharsets.UTF_8);
+ }
+ setFailure(new ProcedureException(errMsg));
+ }
+
+ // result
+ int resultLen = byteBuffer.getInt();
+ if (resultLen > 0) {
+ byte[] resultArr = new byte[resultLen];
+ byteBuffer.get(resultArr);
+ }
+ // has lock
+ if (byteBuffer.get() == 1) {
+ this.lockedWhenLoading();
+ }
+ }
+
+ /**
+ * Deserialize class Name and load class
+ *
+ * @param byteBuffer bytebuffer
+ * @return Procedure
+ */
+ @Deprecated
+ public static void deserializeTypeInfoForCompatibility(ByteBuffer byteBuffer) {
+ int classNameBytesLen = byteBuffer.getInt();
+ byte[] classNameBytes = new byte[classNameBytesLen];
+ byteBuffer.get(classNameBytes);
+ }
+
+ /**
+ * Acquire a lock, user should override it if necessary.
+ *
+ * @param env environment
+ * @return state of lock
+ */
+ protected ProcedureLockState acquireLock(Env env) {
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ /**
+ * Release a lock, user should override it if necessary.
+ *
+ * @param env env
+ */
+ protected void releaseLock(Env env) {
+ // no op
+ }
+
+ /**
+ * Used to keep procedure lock even when the procedure is yielded or suspended.
+ *
+ * @param env env
+ * @return true if hold the lock
+ */
+ protected boolean holdLock(Env env) {
+ return false;
+ }
+
+ /**
+ * To make executor yield between each execution step to give other procedures a chance to run.
+ *
+ * @param env environment
+ * @return return true if yield is allowed.
+ */
+ protected boolean isYieldAfterExecution(Env env) {
+ return false;
+ }
+
+ // -------------------------Internal methods - called by the procedureExecutor------------------
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code execute().
+ *
+ * @param env execute environment
+ * @return sub procedures
+ */
+ protected Procedure<Env>[] doExecute(Env env) throws InterruptedException {
+ try {
+ updateTimestamp();
+ return execute(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code rollback().
+ *
+ * @param env execute environment
+ * @throws IOException ioe
+ * @throws InterruptedException interrupted exception
+ */
+ public void doRollback(Env env) throws IOException, InterruptedException, ProcedureException {
+ try {
+ updateTimestamp();
+ rollback(env);
+ } finally {
+ updateTimestamp();
+ }
+ }
+
+ /**
+ * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock().
+ *
+ * @param env environment
+ * @param store ProcedureStore
+ * @return ProcedureLockState
+ */
+ public final ProcedureLockState doAcquireLock(Env env, IProcedureStore store) {
+ if (lockedWhenLoading) {
+ lockedWhenLoading = false;
+ locked = true;
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+ ProcedureLockState state = acquireLock(env);
+ if (state == ProcedureLockState.LOCK_ACQUIRED) {
+ locked = true;
+ store.update(this);
+ }
+ return state;
+ }
+
+ /**
+ * Presist lock state of the procedure
+ *
+ * @param env environment
+ * @param store ProcedureStore
+ */
+ public final void doReleaseLock(Env env, IProcedureStore store) {
+ locked = false;
+ if (getState() != ProcedureState.ROLLEDBACK) {
+ store.update(this);
+ }
+ releaseLock(env);
+ }
+
+ public final void restoreLock(Env env) {
+ if (!lockedWhenLoading) {
+ LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock", this);
+ return;
+ }
+ if (isFinished()) {
+ LOG.debug("{} is already bypassed, skip acquiring lock.", this);
+ return;
+ }
+ if (getState() == ProcedureState.WAITING && !holdLock(env)) {
+ LOG.debug("{} is in WAITING STATE, and holdLock= false , skip acquiring lock.", this);
+ return;
+ }
+ LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this);
+ acquireLock(env);
+ }
+
+ @Override
+ public String toString() {
+ // Return the simple String presentation of the procedure.
+ return toStringSimpleSB().toString();
+ }
+
+ /**
+ * Build the StringBuilder for the simple form of procedure string.
+ *
+ * @return the StringBuilder
+ */
+ protected StringBuilder toStringSimpleSB() {
+ final StringBuilder sb = new StringBuilder();
+
+ sb.append("pid=");
+ sb.append(getProcId());
+
+ if (hasParent()) {
+ sb.append(", ppid=");
+ sb.append(getParentProcId());
+ }
+
+ /*
+ * TODO
+ * Enable later when this is being used.
+ * Currently owner not used.
+ if (hasOwner()) {
+ sb.append(", owner=");
+ sb.append(getOwner());
+ }*/
+
+ sb.append(", state="); // pState for Procedure State as opposed to any other kind.
+ toStringState(sb);
+
+ // Only print out locked if actually locked. Most of the time it is not.
+ if (this.locked) {
+ sb.append(", locked=").append(locked);
+ }
+ if (hasException()) {
+ sb.append(", exception=" + getException());
+ }
+
+ sb.append("; ");
+ toStringClassDetails(sb);
+
+ return sb;
+ }
+
+ /** Extend the toString() information with more procedure details */
+ public String toStringDetails() {
+ final StringBuilder sb = toStringSimpleSB();
+
+ sb.append(" submittedTime=");
+ sb.append(getSubmittedTime());
+
+ sb.append(", lastUpdate=");
+ sb.append(getLastUpdate());
+
+ final int[] stackIndices = getStackIndexes();
+ if (stackIndices != null) {
+ sb.append("\n");
+ sb.append("stackIndexes=");
+ sb.append(Arrays.toString(stackIndices));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating
+ * generic Procedure State with Procedure particulars.
+ *
+ * @param builder Append current {@link ProcedureState}
+ */
+ protected void toStringState(StringBuilder builder) {
+ builder.append(getState());
+ }
+
+ /**
+ * Extend the toString() information with the procedure details e.g. className and parameters
+ *
+ * @param builder the string builder to use to append the proc specific information
+ */
+ protected void toStringClassDetails(StringBuilder builder) {
+ builder.append(getClass().getName());
+ }
+
+ // ==========================================================================
+ // Those fields are unchanged after initialization.
+ //
+ // Each procedure will get created from the user or during
+ // ProcedureExecutor.start() during the load() phase and then submitted
+ // to the executor. these fields will never be changed after initialization
+ // ==========================================================================
+ public long getProcId() {
+ return procId;
+ }
+
+ public boolean hasParent() {
+ return parentProcId != NO_PROC_ID;
+ }
+
+ public long getParentProcId() {
+ return parentProcId;
+ }
+
+ public long getRootProcId() {
+ return rootProcId;
+ }
+
+ public String getProcType() {
+ return getClass().getSimpleName();
+ }
+
+ public long getSubmittedTime() {
+ return submittedTime;
+ }
+
+ /** Called by the ProcedureExecutor to assign the ID to the newly created procedure. */
+ public void setProcId(long procId) {
+ this.procId = procId;
+ }
+
+ public void setProcRunnable() {
+ this.submittedTime = System.currentTimeMillis();
+ setState(ProcedureState.RUNNABLE);
+ }
+
+ /** Called by the ProcedureExecutor to assign the parent to the newly created procedure. */
+ protected void setParentProcId(long parentProcId) {
+ this.parentProcId = parentProcId;
+ }
+
+ protected void setRootProcId(long rootProcId) {
+ this.rootProcId = rootProcId;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected void setSubmittedTime(long submittedTime) {
+ this.submittedTime = submittedTime;
+ }
+
+ // ==========================================================================
+ // runtime state - timeout related
+ // ==========================================================================
+ /**
+ * @param timeout timeout interval in msec
+ */
+ protected void setTimeout(long timeout) {
+ this.timeout = timeout;
+ }
+
+ public boolean hasTimeout() {
+ return timeout != NO_TIMEOUT;
+ }
+
+ /**
+ * @return the timeout in msec
+ */
+ public long getTimeout() {
+ return timeout;
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected void setLastUpdate(long lastUpdate) {
+ this.lastUpdate = lastUpdate;
+ }
+
+ /** Called by ProcedureExecutor after each time a procedure step is executed. */
+ protected void updateTimestamp() {
+ this.lastUpdate = System.currentTimeMillis();
+ }
+
+ public long getLastUpdate() {
+ return lastUpdate;
+ }
+
+ /**
+ * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set
+ * and the procedure is in the waiting queue.
+ *
+ * @return the timestamp of the next timeout.
+ */
+ protected long getTimeoutTimestamp() {
+ return getLastUpdate() + getTimeout();
+ }
+
+ // ==========================================================================
+ // runtime state
+ // ==========================================================================
+ /**
+ * @return the time elapsed between the last update and the start time of the procedure.
+ */
+ public long elapsedTime() {
+ return getLastUpdate() - getSubmittedTime();
+ }
+
+ /**
+ * @return the serialized result if any, otherwise null
+ */
+ public byte[] getResult() {
+ return result.get();
+ }
+
+ /**
+ * The procedure may leave a "result" on completion.
+ *
+ * @param result the serialized result that will be passed to the client
+ */
+ protected void setResult(byte[] result) {
+ this.result.set(result);
+ }
+
+ /**
+ * Will only be called when loading procedures from procedure store, where we need to record
+ * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)}
+ * to actually acquire the lock.
+ */
+ final void lockedWhenLoading() {
+ this.lockedWhenLoading = true;
+ }
+
+ /**
+ * Can only be called when restarting, before the procedure actually being executed, as after we
+ * actually call the {@link #doAcquireLock(Object, IProcedureStore)} method, we will reset {@link
+ * #lockedWhenLoading} to false.
+ *
+ * <p>Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure
+ * in front of a queue.
+ */
+ public boolean isLockedWhenLoading() {
+ return lockedWhenLoading;
+ }
+
+ // ==============================================================================================
+ // Runtime state, updated every operation by the ProcedureExecutor
+ //
+ // There is always 1 thread at the time operating on the state of the procedure.
+ // The ProcedureExecutor may check and set states, or some Procecedure may
+ // update its own state. but no concurrent updates. we use synchronized here
+ // just because the procedure can get scheduled on different executor threads on each step.
+ // ==============================================================================================
+
+ /**
+ * @return true if the procedure is in a RUNNABLE state.
+ */
+ public synchronized boolean isRunnable() {
+ return state == ProcedureState.RUNNABLE;
+ }
+
+ public synchronized boolean isInitializing() {
+ return state == ProcedureState.INITIALIZING;
+ }
+
+ /**
+ * @return true if the procedure has failed. It may or may not have rolled back.
+ */
+ public synchronized boolean isFailed() {
+ return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /**
+ * @return true if the procedure is finished successfully.
+ */
+ public synchronized boolean isSuccess() {
+ return state == ProcedureState.SUCCESS && !hasException();
+ }
+
+ /**
+ * @return true if the procedure is finished. The Procedure may be completed successfully or
+ * rolledback.
+ */
+ public synchronized boolean isFinished() {
+ return isSuccess() || state == ProcedureState.ROLLEDBACK;
+ }
+
+ /**
+ * @return true if the procedure is waiting for a child to finish or for an external event.
+ */
+ public synchronized boolean isWaiting() {
+ switch (state) {
+ case WAITING:
+ case WAITING_TIMEOUT:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ protected synchronized void setState(final ProcedureState state) {
+ this.state = state;
+ updateTimestamp();
+ }
+
+ public synchronized ProcedureState getState() {
+ return state;
+ }
+
+ protected synchronized void setFailure(final String source, final Throwable cause) {
+ setFailure(new ProcedureException(source, cause));
+ }
+
+ protected synchronized void setFailure(final ProcedureException exception) {
+ this.exception = exception;
+ if (!isFinished()) {
+ setState(ProcedureState.FAILED);
+ }
+ }
+
+ /**
+ * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired.
+ *
+ * @return true to let the framework handle the timeout as abort, false in case the procedure
+ * handled the timeout itself.
+ */
+ protected synchronized boolean setTimeoutFailure(Env env) {
+ if (state == ProcedureState.WAITING_TIMEOUT) {
+ long timeDiff = System.currentTimeMillis() - lastUpdate;
+ setFailure(
+ "ProcedureExecutor",
+ new ProcedureException("Operation timed out after " + timeDiff + " ms."));
+ return true;
+ }
+ return false;
+ }
+
+ public synchronized boolean hasException() {
+ return exception != null;
+ }
+
+ public synchronized ProcedureException getException() {
+ return exception;
+ }
+
+ /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+ protected synchronized void setChildrenLatch(int numChildren) {
+ this.childrenLatch = numChildren;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString()));
+ }
+ }
+
+ /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+ protected synchronized void incChildrenLatch() {
+ // TODO: can this be inferred from the stack? I think so...
+ this.childrenLatch++;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString()));
+ }
+ }
+
+ /** Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. */
+ private synchronized boolean childrenCountDown() {
+ assert childrenLatch > 0 : this;
+ boolean b = --childrenLatch == 0;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString()));
+ }
+ return b;
+ }
+
+ /**
+ * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done.
+ *
+ * @return True if we were able to move procedure to RUNNABLE state.
+ */
+ synchronized boolean tryRunnable() {
+ // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT
+ if (getState() == ProcedureState.WAITING && childrenCountDown()) {
+ setState(ProcedureState.RUNNABLE);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ protected synchronized boolean hasChildren() {
+ return childrenLatch > 0;
+ }
+
+ protected synchronized int getChildrenLatch() {
+ return childrenLatch;
+ }
+
+ /**
+ * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index
+ * positions.
+ */
+ protected synchronized void addStackIndex(final int index) {
+ if (stackIndexes == null) {
+ stackIndexes = new int[] {index};
+ } else {
+ int count = stackIndexes.length;
+ stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
+ stackIndexes[count] = index;
+ }
+ }
+
+ protected synchronized boolean removeStackIndex() {
+ if (stackIndexes != null && stackIndexes.length > 1) {
+ stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
+ return false;
+ } else {
+ stackIndexes = null;
+ return true;
+ }
+ }
+
+ /**
+ * Called on store load to initialize the Procedure internals after the creation/deserialization.
+ */
+ protected synchronized void setStackIndexes(final List<Integer> stackIndexes) {
+ this.stackIndexes = new int[stackIndexes.size()];
+ for (int i = 0; i < this.stackIndexes.length; ++i) {
+ this.stackIndexes[i] = stackIndexes.get(i);
+ }
+ }
+
+ protected synchronized boolean wasExecuted() {
+ return stackIndexes != null;
+ }
+
+ protected synchronized int[] getStackIndexes() {
+ return stackIndexes;
+ }
+
+ public void setRootProcedureId(long rootProcedureId) {
+ this.rootProcId = rootProcedureId;
+ }
+
+ /**
+ * @param a the first procedure to be compared.
+ * @param b the second procedure to be compared.
+ * @return true if the two procedures have the same parent
+ */
+ public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) {
+ return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId());
+ }
+
+ @Override
+ public int compareTo(Procedure<Env> other) {
+ return Long.compare(getProcId(), other.getProcId());
+ }
+
+ /**
+ * This function will be called just when procedure is submitted for execution.
+ *
+ * @param env The environment passed to the procedure executor
+ */
+ protected void updateMetricsOnSubmit(Env env) {
+ if (env instanceof ConfigNodeProcedureEnv) {
+ ((ConfigNodeProcedureEnv) env)
+ .getConfigManager()
+ .getProcedureManager()
+ .getProcedureMetrics()
+ .updateMetricsOnSubmit(getProcType());
+ }
+ }
+
+ /**
+ * This function will be called just after procedure execution is finished. Override this method
+ * to update metrics at the end of the procedure. The default implementation adds runtime of a
+ * procedure to a time histogram for successfully completed procedures. Increments failed counter
+ * for failed procedures.
+ *
+ * @param env The environment passed to the procedure executor
+ * @param runtime Runtime of the procedure in milliseconds
+ * @param success true if procedure is completed successfully
+ */
+ protected void updateMetricsOnFinish(Env env, long runtime, boolean success) {
+ if (env instanceof ConfigNodeProcedureEnv) {
+ ((ConfigNodeProcedureEnv) env)
+ .getConfigManager()
+ .getProcedureManager()
+ .getProcedureMetrics()
+ .updateMetricsOnFinish(getProcType(), runtime, success);
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
index 4c7421885f7..0d8368583b4 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
@@ -1,941 +1,940 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.commons.concurrent.ThreadName;
-1import org.apache.iotdb.commons.utils.TestOnly;
-1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-1import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
-1import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
-1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-1
-1import com.google.common.base.Preconditions;
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.io.IOException;
-1import java.util.ArrayDeque;
-1import java.util.ArrayList;
-1import java.util.Arrays;
-1import java.util.Deque;
-1import java.util.HashSet;
-1import java.util.List;
-1import java.util.Objects;
-1import java.util.Set;
-1import java.util.concurrent.ConcurrentHashMap;
-1import java.util.concurrent.CopyOnWriteArrayList;
-1import java.util.concurrent.TimeUnit;
-1import java.util.concurrent.atomic.AtomicBoolean;
-1import java.util.concurrent.atomic.AtomicInteger;
-1import java.util.concurrent.atomic.AtomicLong;
-1import java.util.concurrent.atomic.AtomicReference;
-1
-1import static org.apache.iotdb.confignode.procedure.Procedure.NO_PROC_ID;
-1
-1public class ProcedureExecutor<Env> {
-1 private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);
-1
-1 private final ConcurrentHashMap<Long, CompletedProcedureContainer<Env>> completed =
-1 new ConcurrentHashMap<>();
-1
-1 private final ConcurrentHashMap<Long, RootProcedureStack<Env>> rollbackStack =
-1 new ConcurrentHashMap<>();
-1
-1 private final ConcurrentHashMap<Long, Procedure<Env>> procedures = new ConcurrentHashMap<>();
-1
-1 private ThreadGroup threadGroup;
-1
-1 private CopyOnWriteArrayList<WorkerThread> workerThreads;
-1
-1 private TimeoutExecutorThread<Env> timeoutExecutor;
-1
-1 private TimeoutExecutorThread<Env> workerMonitorExecutor;
-1
-1 private int corePoolSize;
-1 private int maxPoolSize;
-1
-1 private final ProcedureScheduler scheduler;
-1
-1 private final AtomicLong workId = new AtomicLong(0);
-1 private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
-1 private final AtomicBoolean running = new AtomicBoolean(false);
-1 private final Env environment;
-1 private final IProcedureStore<Env> store;
-1
-1 public ProcedureExecutor(
-1 final Env environment, final IProcedureStore<Env> store, final ProcedureScheduler scheduler) {
-1 this.environment = environment;
-1 this.scheduler = scheduler;
-1 this.store = store;
-1 }
-1
-1 @TestOnly
-1 public ProcedureExecutor(final Env environment, final IProcedureStore<Env> store) {
-1 this(environment, store, new SimpleProcedureScheduler());
-1 }
-1
-1 public void init(int numThreads) {
-1 this.corePoolSize = numThreads;
-1 this.maxPoolSize = 10 * numThreads;
-1 this.threadGroup = new ThreadGroup(ThreadName.CONFIG_NODE_PROCEDURE_WORKER.getName());
-1 this.timeoutExecutor =
-1 new TimeoutExecutorThread<>(
-1 this, threadGroup, ThreadName.CONFIG_NODE_TIMEOUT_EXECUTOR.getName());
-1 this.workerMonitorExecutor =
-1 new TimeoutExecutorThread<>(
-1 this, threadGroup, ThreadName.CONFIG_NODE_WORKER_THREAD_MONITOR.getName());
-1 workId.set(0);
-1 workerThreads = new CopyOnWriteArrayList<>();
-1 for (int i = 0; i < corePoolSize; i++) {
-1 workerThreads.add(new WorkerThread(threadGroup));
-1 }
-1 // Add worker monitor
-1 workerMonitorExecutor.add(new WorkerMonitor());
-1
-1 scheduler.start();
-1 recover();
-1 }
-1
-1 private void recover() {
-1 // 1.Build rollback stack
-1 List<Procedure<Env>> procedureList = getProcedureListFromDifferentVersion();
-1 // Load procedure wal file
-1 for (Procedure<Env> proc : procedureList) {
-1 if (proc.isFinished()) {
-1 completed.putIfAbsent(proc.getProcId(), new CompletedProcedureContainer<>(proc));
-1 } else {
-1 if (!proc.hasParent()) {
-1 rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
-1 }
-1 procedures.putIfAbsent(proc.getProcId(), proc);
-1 }
-1 }
-1 List<Procedure<Env>> runnableList = new ArrayList<>();
-1 List<Procedure<Env>> failedList = new ArrayList<>();
-1 List<Procedure<Env>> waitingList = new ArrayList<>();
-1 List<Procedure<Env>> waitingTimeoutList = new ArrayList<>();
-1 for (Procedure<Env> proc : procedureList) {
-1 if (proc.isFinished() && !proc.hasParent()) {
-1 continue;
-1 }
-1 long rootProcedureId = getRootProcedureId(proc);
-1 if (proc.hasParent()) {
-1 Procedure<Env> parent = procedures.get(proc.getParentProcId());
-1 if (parent != null && !proc.isFinished()) {
-1 parent.incChildrenLatch();
-1 }
-1 }
-1 RootProcedureStack<Env> rootStack = rollbackStack.get(rootProcedureId);
-1 if (rootStack != null) {
-1 rootStack.loadStack(proc);
-1 }
-1 proc.setRootProcedureId(rootProcedureId);
-1 switch (proc.getState()) {
-1 case RUNNABLE:
-1 runnableList.add(proc);
-1 break;
-1 case FAILED:
-1 failedList.add(proc);
-1 break;
-1 case WAITING:
-1 waitingList.add(proc);
-1 break;
-1 case WAITING_TIMEOUT:
-1 waitingTimeoutList.add(proc);
-1 break;
-1 case ROLLEDBACK:
-1 case INITIALIZING:
-1 LOG.error("Unexpected state:{} for {}", proc.getState(), proc);
-1 throw new UnsupportedOperationException("Unexpected state");
-1 default:
-1 break;
-1 }
-1 }
-1
-1 waitingList.forEach(
-1 procedure -> {
-1 if (!procedure.hasChildren()) {
-1 // Normally, WAITING procedures should be wakened by its children.
-1 // But, there is a case that, all the children are successful, and before
-1 // they can wake up their parent procedure, the master was killed.
-1 // So, during recovering the procedures from ProcedureWal, its children
-1 // are not loaded because of their SUCCESS state.
-1 // So we need to continue to run this WAITING procedure. Before
-1 // executing, we need to set its state to RUNNABLE.
-1 procedure.setState(ProcedureState.RUNNABLE);
-1 runnableList.add(procedure);
-1 }
-1 });
-1 restoreLocks();
-1
-1 waitingTimeoutList.forEach(timeoutExecutor::add);
-1
-1 failedList.forEach(scheduler::addBack);
-1 runnableList.forEach(scheduler::addBack);
-1 scheduler.signalAll();
-1 }
-1
-1 private List<Procedure<Env>> getProcedureListFromDifferentVersion() {
-1 if (store.isOldVersionProcedureStore()) {
-1 LOG.info("Old procedure directory detected, upgrade beginning...");
-1 return store.load();
-1 } else {
-1 return store.getProcedures();
-1 }
-1 }
-1
-1 /**
-1 * Helper to look up the root Procedure ID.
-1 *
-1 * @param proc given a specified procedure.
-1 */
-1 Long getRootProcedureId(Procedure<Env> proc) {
-1 while (proc.hasParent()) {
-1 proc = procedures.get(proc.getParentProcId());
-1 if (proc == null) {
-1 return NO_PROC_ID;
-1 }
-1 }
-1 return proc.getProcId();
-1 }
-1
-1 private void releaseLock(Procedure<Env> procedure, boolean force) {
-1 if (force || !procedure.holdLock(this.environment) || procedure.isFinished()) {
-1 procedure.doReleaseLock(this.environment, store);
-1 }
-1 }
-1
-1 private void restoreLock(Procedure procedure, Set<Long> restored) {
-1 procedure.restoreLock(environment);
-1 restored.add(procedure.getProcId());
-1 }
-1
-1 private void restoreLocks(Deque<Procedure<Env>> stack, Set<Long> restored) {
-1 while (!stack.isEmpty()) {
-1 restoreLock(stack.pop(), restored);
-1 }
-1 }
-1
-1 private void restoreLocks() {
-1 Set<Long> restored = new HashSet<>();
-1 Deque<Procedure<Env>> stack = new ArrayDeque<>();
-1 procedures
-1 .values()
-1 .forEach(
-1 procedure -> {
-1 while (procedure != null) {
-1 if (restored.contains(procedure.getProcId())) {
-1 restoreLocks(stack, restored);
-1 return;
-1 }
-1 if (!procedure.hasParent()) {
-1 restoreLock(procedure, restored);
-1 restoreLocks(stack, restored);
-1 return;
-1 }
-1 stack.push(procedure);
-1 procedure = procedures.get(procedure.getParentProcId());
-1 }
-1 });
-1 }
-1
-1 public void startWorkers() {
-1 if (!running.compareAndSet(false, true)) {
-1 LOG.warn("Already running");
-1 return;
-1 }
-1 timeoutExecutor.start();
-1 workerMonitorExecutor.start();
-1 for (WorkerThread workerThread : workerThreads) {
-1 workerThread.start();
-1 }
-1 LOG.info("{} procedure workers are started.", workerThreads.size());
-1 }
-1
-1 public void startCompletedCleaner(long cleanTimeInterval, long cleanEvictTTL) {
-1 addInternalProcedure(
-1 new CompletedProcedureRecycler(store, completed, cleanTimeInterval, cleanEvictTTL));
-1 }
-1
-1 public void addInternalProcedure(InternalProcedure interalProcedure) {
-1 if (interalProcedure == null) {
-1 return;
-1 }
-1 interalProcedure.setState(ProcedureState.WAITING_TIMEOUT);
-1 timeoutExecutor.add(interalProcedure);
-1 }
-1
-1 public boolean removeInternalProcedure(InternalProcedure internalProcedure) {
-1 if (internalProcedure == null) {
-1 return true;
-1 }
-1 internalProcedure.setState(ProcedureState.SUCCESS);
-1 return timeoutExecutor.remove(internalProcedure);
-1 }
-1
-1 /**
-1 * Executes procedure
-1 *
-1 * <p>Calls doExecute() if success and return subprocedures submit sub procs set the state to
-1 * WAITING, wait for all sub procs completed. else if no sub procs procedure completed
-1 * successfully set procedure's parent to RUNNABLE in case of failure start rollback of the
-1 * procedure.
-1 *
-1 * @param proc procedure
-1 */
-1 private void executeProcedure(Procedure<Env> proc) {
-1 if (proc.isFinished()) {
-1 LOG.debug("{} is already finished.", proc);
-1 return;
-1 }
-1 final Long rootProcId = getRootProcedureId(proc);
-1 if (rootProcId == null) {
-1 LOG.warn("Rollback because parent is done/rolledback, proc is {}", proc);
-1 executeRollback(proc);
-1 return;
-1 }
-1 RootProcedureStack<Env> rootProcStack = rollbackStack.get(rootProcId);
-1 if (rootProcStack == null) {
-1 LOG.warn("Rollback stack is null for {}", proc.getProcId());
-1 return;
-1 }
-1 ProcedureLockState lockState = null;
-1 try {
-1 do {
-1 if (!rootProcStack.acquire()) {
-1 if (rootProcStack.setRollback()) {
-1 lockState = executeRootStackRollback(rootProcId, rootProcStack);
-1 switch (lockState) {
-1 case LOCK_ACQUIRED:
-1 break;
-1 case LOCK_EVENT_WAIT:
-1 LOG.info("LOCK_EVENT_WAIT rollback {}", proc);
-1 rootProcStack.unsetRollback();
-1 break;
-1 case LOCK_YIELD_WAIT:
-1 rootProcStack.unsetRollback();
-1 scheduler.yield(proc);
-1 break;
-1 default:
-1 throw new UnsupportedOperationException();
-1 }
-1 } else {
-1 if (!proc.wasExecuted()) {
-1 switch (executeRollback(proc)) {
-1 case LOCK_ACQUIRED:
-1 break;
-1 case LOCK_EVENT_WAIT:
-1 LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc);
-1 break;
-1 case LOCK_YIELD_WAIT:
-1 scheduler.yield(proc);
-1 break;
-1 default:
-1 throw new UnsupportedOperationException();
-1 }
-1 }
-1 }
-1 break;
-1 }
-1 lockState = acquireLock(proc);
-1 switch (lockState) {
-1 case LOCK_ACQUIRED:
-1 executeProcedure(rootProcStack, proc);
-1 break;
-1 case LOCK_YIELD_WAIT:
-1 case LOCK_EVENT_WAIT:
-1 LOG.info("{} lockstate is {}", proc, lockState);
-1 break;
-1 default:
-1 throw new UnsupportedOperationException();
-1 }
-1 rootProcStack.release();
-1
-1 if (proc.isSuccess()) {
-1 // update metrics on finishing the procedure
-1 proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);
-1 LOG.debug("{} finished in {}ms successfully.", proc, proc.elapsedTime());
-1 if (proc.getProcId() == rootProcId) {
-1 rootProcedureCleanup(proc);
-1 } else {
-1 executeCompletionCleanup(proc);
-1 }
-1 return;
-1 }
-1
-1 } while (rootProcStack.isFailed());
-1 } finally {
-1 // Only after procedure has completed execution can it be allowed to be rescheduled to prevent
-1 // data races
-1 if (Objects.equals(lockState, ProcedureLockState.LOCK_EVENT_WAIT)) {
-1 LOG.info("procedureId {} wait for lock.", proc.getProcId());
-1 ((ConfigNodeProcedureEnv) this.environment).getNodeLock().waitProcedure(proc);
-1 }
-1 }
-1 }
-1
-1 /**
-1 * execute procedure and submit its children
-1 *
-1 * @param rootProcStack procedure's root proc stack
-1 * @param proc procedure
-1 */
-1 private void executeProcedure(RootProcedureStack rootProcStack, Procedure<Env> proc) {
-1 if (proc.getState() != ProcedureState.RUNNABLE) {
-1 LOG.error(
-1 "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}", proc);
-1 return;
-1 }
-1 boolean reExecute;
-1
-1 Procedure<Env>[] subprocs = null;
-1 do {
-1 reExecute = false;
-1 try {
-1 subprocs = proc.doExecute(this.environment);
-1 if (subprocs != null && subprocs.length == 0) {
-1 subprocs = null;
-1 }
-1 } catch (InterruptedException e) {
-1 LOG.warn("Interrupt during execution, suspend or retry it later.", e);
-1 yieldProcedure(proc);
-1 } catch (Throwable e) {
-1 LOG.error("CODE-BUG:{}", proc, e);
-1 proc.setFailure(new ProcedureException(e.getMessage(), e));
-1 }
-1
-1 if (!proc.isFailed()) {
-1 if (subprocs != null) {
-1 if (subprocs.length == 1 && subprocs[0] == proc) {
-1 subprocs = null;
-1 reExecute = true;
-1 } else {
-1 subprocs = initializeChildren(rootProcStack, proc, subprocs);
-1 LOG.info("Initialized sub procs:{}", Arrays.toString(subprocs));
-1 }
-1 } else if (proc.getState() == ProcedureState.WAITING_TIMEOUT) {
-1 LOG.info("Added into timeoutExecutor {}", proc);
-1 } else {
-1 proc.setState(ProcedureState.SUCCESS);
-1 }
-1 }
-1 // add procedure into rollback stack.
-1 rootProcStack.addRollbackStep(proc);
-1
-1 updateStoreOnExecution(rootProcStack, proc, subprocs);
-1
-1 if (!store.isRunning()) {
-1 return;
-1 }
-1
-1 if (proc.isRunnable() && proc.isYieldAfterExecution(this.environment)) {
-1 yieldProcedure(proc);
-1 return;
-1 }
-1 } while (reExecute);
-1
-1 if (subprocs != null && !proc.isFailed()) {
-1 submitChildrenProcedures(subprocs);
-1 }
-1
-1 releaseLock(proc, false);
-1 if (proc.isFinished() && proc.hasParent()) {
-1 countDownChildren(rootProcStack, proc);
-1 }
-1 }
-1
-1 /**
-1 * Serve as a countdown latch to check whether all children have already completed.
-1 *
-1 * @param rootProcStack root procedure stack
-1 * @param proc proc
-1 */
-1 private void countDownChildren(RootProcedureStack rootProcStack, Procedure<Env> proc) {
-1 Procedure<Env> parent = procedures.get(proc.getParentProcId());
-1 if (parent == null && rootProcStack.isRollingback()) {
-1 return;
-1 }
-1 if (parent != null && parent.tryRunnable()) {
-1 // If success, means all its children have completed, move parent to front of the queue.
-1 store.update(parent);
-1 scheduler.addFront(parent);
-1 LOG.info(
-1 "Finished subprocedure pid={}, resume processing ppid={}",
-1 proc.getProcId(),
-1 parent.getProcId());
-1 }
-1 }
-1
-1 /**
-1 * Submit children procedures.
-1 *
-1 * @param subprocs children procedures
-1 */
-1 private void submitChildrenProcedures(Procedure<Env>[] subprocs) {
-1 for (Procedure<Env> subproc : subprocs) {
-1 subproc.updateMetricsOnSubmit(getEnvironment());
-1 procedures.put(subproc.getProcId(), subproc);
-1 scheduler.addFront(subproc);
-1 LOG.info("Sub-Procedure pid={} has been submitted", subproc.getProcId());
-1 }
-1 }
-1
-1 private void updateStoreOnExecution(
-1 RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
-1 if (subprocs != null && !proc.isFailed()) {
-1 if (LOG.isDebugEnabled()) {
-1 LOG.debug("Stored {}, children {}", proc, Arrays.toString(subprocs));
-1 }
-1 store.update(subprocs);
-1 } else {
-1 LOG.debug("Store update {}", proc);
-1 if (proc.isFinished() && !proc.hasParent()) {
-1 final long[] childProcIds = rootProcStack.getSubprocedureIds();
-1 if (childProcIds != null) {
-1 store.delete(childProcIds);
-1 for (long childProcId : childProcIds) {
-1 procedures.remove(childProcId);
-1 }
-1 } else {
-1 store.update(proc);
-1 }
-1 } else {
-1 store.update(proc);
-1 }
-1 }
-1 }
-1
-1 private Procedure<Env>[] initializeChildren(
-1 RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
-1 final long rootProcedureId = getRootProcedureId(proc);
-1 for (int i = 0; i < subprocs.length; i++) {
-1 Procedure<Env> subproc = subprocs[i];
-1 if (subproc == null) {
-1 String errMsg = "subproc[" + i + "] is null, aborting procedure";
-1 proc.setFailure(new ProcedureException((errMsg), new IllegalArgumentException(errMsg)));
-1 return null;
-1 }
-1 subproc.setParentProcId(proc.getProcId());
-1 subproc.setRootProcId(rootProcedureId);
-1 subproc.setProcId(store.getNextProcId());
-1 subproc.setProcRunnable();
-1 rootProcStack.addSubProcedure(subproc);
-1 }
-1
-1 if (!proc.isFailed()) {
-1 proc.setChildrenLatch(subprocs.length);
-1 switch (proc.getState()) {
-1 case RUNNABLE:
-1 proc.setState(ProcedureState.WAITING);
-1 break;
-1 case WAITING_TIMEOUT:
-1 timeoutExecutor.add(proc);
-1 break;
-1 default:
-1 break;
-1 }
-1 }
-1 return subprocs;
-1 }
-1
-1 private void yieldProcedure(Procedure<Env> proc) {
-1 releaseLock(proc, false);
-1 scheduler.yield(proc);
-1 }
-1
-1 /**
-1 * Rollback full root procedure stack.
-1 *
-1 * @param rootProcId root procedure id
-1 * @param procedureStack root procedure stack
-1 * @return lock state
-1 */
-1 private ProcedureLockState executeRootStackRollback(
-1 Long rootProcId, RootProcedureStack procedureStack) {
-1 Procedure<Env> rootProcedure = procedures.get(rootProcId);
-1 ProcedureException exception = rootProcedure.getException();
-1 if (exception == null) {
-1 exception = procedureStack.getException();
-1 rootProcedure.setFailure(exception);
-1 store.update(rootProcedure);
-1 }
-1 List<Procedure<Env>> subprocStack = procedureStack.getSubproceduresStack();
-1 int stackTail = subprocStack.size();
-1 while (stackTail-- > 0) {
-1 Procedure<Env> procedure = subprocStack.get(stackTail);
-1 if (procedure.isSuccess()) {
-1 subprocStack.remove(stackTail);
-1 cleanupAfterRollback(procedure);
-1 continue;
-1 }
-1 ProcedureLockState lockState = acquireLock(procedure);
-1 if (lockState != ProcedureLockState.LOCK_ACQUIRED) {
-1 return lockState;
-1 }
-1 lockState = executeRollback(procedure);
-1 releaseLock(procedure, false);
-1
-1 boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED;
-1 abortRollback |= !isRunning() || !store.isRunning();
-1 if (abortRollback) {
-1 return lockState;
-1 }
-1
-1 if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) {
-1 return ProcedureLockState.LOCK_YIELD_WAIT;
-1 }
-1
-1 if (procedure != rootProcedure) {
-1 executeCompletionCleanup(procedure);
-1 }
-1 }
-1
-1 LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime());
-1 rootProcedureCleanup(rootProcedure);
-1 return ProcedureLockState.LOCK_ACQUIRED;
-1 }
-1
-1 private ProcedureLockState acquireLock(Procedure<Env> proc) {
-1 if (proc.hasLock()) {
-1 return ProcedureLockState.LOCK_ACQUIRED;
-1 }
-1 return proc.doAcquireLock(this.environment, store);
-1 }
-1
-1 /**
-1 * do execute defined in procedure and then update store or remove completely in case it is a
-1 * child.
-1 *
-1 * @param procedure procedure
-1 * @return procedure lock state
-1 */
-1 private ProcedureLockState executeRollback(Procedure<Env> procedure) {
-1 try {
-1 procedure.doRollback(this.environment);
-1 } catch (IOException e) {
-1 LOG.error("Roll back failed for {}", procedure, e);
-1 } catch (InterruptedException e) {
-1 LOG.warn("Interrupted exception occured for {}", procedure, e);
-1 } catch (Throwable t) {
-1 LOG.error("CODE-BUG: runtime exception for {}", procedure, t);
-1 }
-1 cleanupAfterRollback(procedure);
-1 return ProcedureLockState.LOCK_ACQUIRED;
-1 }
-1
-1 private void cleanupAfterRollback(Procedure<Env> procedure) {
-1 if (procedure.removeStackIndex()) {
-1 if (!procedure.isSuccess()) {
-1 procedure.setState(ProcedureState.ROLLEDBACK);
-1 }
-1
-1 // update metrics on finishing the procedure (fail)
-1 procedure.updateMetricsOnFinish(getEnvironment(), procedure.elapsedTime(), false);
-1
-1 if (procedure.hasParent()) {
-1 store.delete(procedure.getProcId());
-1 procedures.remove(procedure.getProcId());
-1 } else {
-1 final long[] childProcIds = rollbackStack.get(procedure.getProcId()).getSubprocedureIds();
-1 if (childProcIds != null) {
-1 store.delete(childProcIds);
-1 } else {
-1 store.update(procedure);
-1 }
-1 }
-1 } else {
-1 store.update(procedure);
-1 }
-1 }
-1
-1 private void executeCompletionCleanup(Procedure<Env> proc) {
-1 if (proc.hasLock()) {
-1 releaseLock(proc, true);
-1 }
-1 }
-1
-1 private void rootProcedureCleanup(Procedure<Env> proc) {
-1 executeCompletionCleanup(proc);
-1 CompletedProcedureContainer<Env> retainer = new CompletedProcedureContainer<>(proc);
-1 completed.put(proc.getProcId(), retainer);
-1 rollbackStack.remove(proc.getProcId());
-1 procedures.remove(proc.getProcId());
-1 }
-1
-1 /**
-1 * Add a Procedure to executor.
-1 *
-1 * @param procedure procedure
-1 * @return procedure id
-1 */
-1 private long pushProcedure(Procedure<Env> procedure) {
-1 final long currentProcId = procedure.getProcId();
-1 // Update metrics on start of a procedure
-1 procedure.updateMetricsOnSubmit(getEnvironment());
-1 RootProcedureStack<Env> stack = new RootProcedureStack<>();
-1 rollbackStack.put(currentProcId, stack);
-1 procedures.put(currentProcId, procedure);
-1 scheduler.addBack(procedure);
-1 return procedure.getProcId();
-1 }
-1
-1 private class WorkerThread extends StoppableThread {
-1 private final AtomicLong startTime = new AtomicLong(Long.MAX_VALUE);
-1 private final AtomicReference<Procedure<Env>> activeProcedure = new AtomicReference<>();
-1 protected long keepAliveTime = -1;
-1
-1 public WorkerThread(ThreadGroup threadGroup) {
-1 this(threadGroup, "ProcedureCoreWorker-");
-1 }
-1
-1 public WorkerThread(ThreadGroup threadGroup, String prefix) {
-1 super(threadGroup, prefix + workId.incrementAndGet());
-1 setDaemon(true);
-1 }
-1
-1 @Override
-1 public void sendStopSignal() {
-1 scheduler.signalAll();
-1 }
-1
-1 @Override
-1 public void run() {
-1 long lastUpdated = System.currentTimeMillis();
-1 try {
-1 while (isRunning() && keepAlive(lastUpdated)) {
-1 Procedure<Env> procedure = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);
-1 if (procedure == null) {
-1 Thread.sleep(1000);
-1 continue;
-1 }
-1 this.activeProcedure.set(procedure);
-1 activeExecutorCount.incrementAndGet();
-1 startTime.set(System.currentTimeMillis());
-1 executeProcedure(procedure);
-1 activeExecutorCount.decrementAndGet();
-1 LOG.trace(
-1 "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get());
-1 this.activeProcedure.set(null);
-1 lastUpdated = System.currentTimeMillis();
-1 startTime.set(lastUpdated);
-1 }
-1
-1 } catch (Exception e) {
-1 if (this.activeProcedure.get() != null) {
-1 LOG.warn(
-1 "Exception happened when worker {} execute procedure {}",
-1 getName(),
-1 this.activeProcedure.get(),
-1 e);
-1 }
-1 } finally {
-1 LOG.info("Procedure worker {} terminated.", getName());
-1 }
-1 workerThreads.remove(this);
-1 }
-1
-1 protected boolean keepAlive(long lastUpdated) {
-1 return true;
-1 }
-1
-1 @Override
-1 public String toString() {
-1 Procedure<?> p = this.activeProcedure.get();
-1 return getName() + "(pid=" + (p == null ? NO_PROC_ID : p.getProcId() + ")");
-1 }
-1
-1 /**
-1 * @return the time since the current procedure is running
-1 */
-1 public long getCurrentRunTime() {
-1 return System.currentTimeMillis() - startTime.get();
-1 }
-1 }
-1
-1 // A temporary worker thread will be launched when too many core workers are stuck.
-1 // They will timeout after keepAliveTime if there is no procedure to run.
-1 private final class TemporaryWorkerThread extends WorkerThread {
-1
-1 public TemporaryWorkerThread(ThreadGroup group) {
-1 super(group, "ProcedureTemporaryWorker-");
-1 this.keepAliveTime = TimeUnit.SECONDS.toMillis(10);
-1 }
-1
-1 @Override
-1 protected boolean keepAlive(long lastUpdate) {
-1 return System.currentTimeMillis() - lastUpdate < keepAliveTime;
-1 }
-1 }
-1
-1 private final class WorkerMonitor extends InternalProcedure<Env> {
-1 private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 30000; // 30sec
-1
-1 private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 60000; // 60sec
-1
-1 private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck
-1
-1 public WorkerMonitor() {
-1 super(DEFAULT_WORKER_MONITOR_INTERVAL);
-1 updateTimestamp();
-1 }
-1
-1 private int calculateRunningAndStuckWorkers() {
-1 // Check if any of the worker is stuck
-1 int runningCount = 0, stuckCount = 0;
-1 for (WorkerThread worker : workerThreads) {
-1 final Procedure<?> proc = worker.activeProcedure.get();
-1 if (proc == null) {
-1 continue;
-1 }
-1 runningCount++;
-1 // WARN the worker is stuck
-1 if (worker.getCurrentRunTime() > DEFAULT_WORKER_STUCK_THRESHOLD) {
-1 stuckCount++;
-1 LOG.warn(
-1 "Worker stuck {}({}), run time {} ms",
-1 worker,
-1 proc.getProcType(),
-1 worker.getCurrentRunTime());
-1 }
-1 LOG.info(
-1 "Procedure workers: {} is running, {} is running and stuck", runningCount, stuckCount);
-1 }
-1 return stuckCount;
-1 }
-1
-1 private void checkThreadCount(final int stuckCount) {
-1 // Nothing to do if there are no runnable tasks
-1 if (stuckCount < 1 || !scheduler.hasRunnables()) {
-1 return;
-1 }
-1 // Add a new thread if the worker stuck percentage exceed the threshold limit
-1 // and every handler is active.
-1 final float stuckPerc = ((float) stuckCount) / workerThreads.size();
-1 // Let's add new worker thread more aggressively, as they will timeout finally if there is no
-1 // work to do.
-1 if (stuckPerc >= DEFAULT_WORKER_ADD_STUCK_PERCENTAGE && workerThreads.size() < maxPoolSize) {
-1 final TemporaryWorkerThread worker = new TemporaryWorkerThread(threadGroup);
-1 workerThreads.add(worker);
-1 worker.start();
-1 LOG.debug("Added new worker thread {}", worker);
-1 }
-1 }
-1
-1 @Override
-1 protected void periodicExecute(Env env) {
-1 final int stuckCount = calculateRunningAndStuckWorkers();
-1 checkThreadCount(stuckCount);
-1 updateTimestamp();
-1 }
-1 }
-1
-1 public int getWorkerThreadCount() {
-1 return workerThreads.size();
-1 }
-1
-1 public long getActiveWorkerThreadCount() {
-1 return workerThreads.stream().filter(worker -> worker.activeProcedure.get() != null).count();
-1 }
-1
-1 public boolean isRunning() {
-1 return running.get();
-1 }
-1
-1 public void stop() {
-1 if (!running.getAndSet(false)) {
-1 return;
-1 }
-1 LOG.info("Stopping");
-1 scheduler.stop();
-1 timeoutExecutor.sendStopSignal();
-1 }
-1
-1 public void join() {
-1 timeoutExecutor.awaitTermination();
-1 workerMonitorExecutor.awaitTermination();
-1 for (WorkerThread workerThread : workerThreads) {
-1 workerThread.awaitTermination();
-1 }
-1 try {
-1 threadGroup.destroy();
-1 } catch (IllegalThreadStateException e) {
-1 LOG.warn(
-1 "ProcedureExecutor threadGroup {} contains running threads which are used by non-procedure module.",
-1 this.threadGroup);
-1 this.threadGroup.list();
-1 }
-1 }
-1
-1 public boolean isStarted(long procId) {
-1 Procedure<Env> procedure = procedures.get(procId);
-1 if (procedure == null) {
-1 return completed.get(procId) != null;
-1 }
-1 return procedure.wasExecuted();
-1 }
-1
-1 public boolean isFinished(final long procId) {
-1 return !procedures.containsKey(procId);
-1 }
-1
-1 public ConcurrentHashMap<Long, Procedure<Env>> getProcedures() {
-1 return procedures;
-1 }
-1
-1 // -----------------------------CLIENT IMPLEMENTATION-----------------------------------
-1 /**
-1 * Submit a new root-procedure to the executor, called by client.
-1 *
-1 * @param procedure root procedure
-1 * @return procedure id
-1 */
-1 public long submitProcedure(Procedure<Env> procedure) {
-1 Preconditions.checkArgument(procedure.getState() == ProcedureState.INITIALIZING);
-1 Preconditions.checkArgument(!procedure.hasParent(), "Unexpected parent", procedure);
-1 // Initialize the procedure
-1 procedure.setProcId(store.getNextProcId());
-1 procedure.setProcRunnable();
-1 // Commit the transaction
-1 store.update(procedure);
-1 LOG.debug("{} is stored.", procedure);
-1 // Add the procedure to the executor
-1 return pushProcedure(procedure);
-1 }
-1
-1 public ProcedureScheduler getScheduler() {
-1 return scheduler;
-1 }
-1
-1 public Env getEnvironment() {
-1 return environment;
-1 }
-1
-1 public IProcedureStore getStore() {
-1 return store;
-1 }
-1
-1 public RootProcedureStack<Env> getRollbackStack(long rootProcId) {
-1 return rollbackStack.get(rootProcId);
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.commons.concurrent.ThreadName;
+import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
+import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
+import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.apache.iotdb.confignode.procedure.Procedure.NO_PROC_ID;
+
+public class ProcedureExecutor<Env> {
+ private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);
+
+ private final ConcurrentHashMap<Long, CompletedProcedureContainer<Env>> completed =
+ new ConcurrentHashMap<>();
+
+ private final ConcurrentHashMap<Long, RootProcedureStack<Env>> rollbackStack =
+ new ConcurrentHashMap<>();
+
+ private final ConcurrentHashMap<Long, Procedure<Env>> procedures = new ConcurrentHashMap<>();
+
+ private ThreadGroup threadGroup;
+
+ private CopyOnWriteArrayList<WorkerThread> workerThreads;
+
+ private TimeoutExecutorThread<Env> timeoutExecutor;
+
+ private TimeoutExecutorThread<Env> workerMonitorExecutor;
+
+ private int corePoolSize;
+ private int maxPoolSize;
+
+ private final ProcedureScheduler scheduler;
+
+ private final AtomicLong workId = new AtomicLong(0);
+ private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
+ private final AtomicBoolean running = new AtomicBoolean(false);
+ private final Env environment;
+ private final IProcedureStore<Env> store;
+
+ public ProcedureExecutor(
+ final Env environment, final IProcedureStore<Env> store, final ProcedureScheduler scheduler) {
+ this.environment = environment;
+ this.scheduler = scheduler;
+ this.store = store;
+ }
+
+ @TestOnly
+ public ProcedureExecutor(final Env environment, final IProcedureStore<Env> store) {
+ this(environment, store, new SimpleProcedureScheduler());
+ }
+
+ public void init(int numThreads) {
+ this.corePoolSize = numThreads;
+ this.maxPoolSize = 10 * numThreads;
+ this.threadGroup = new ThreadGroup(ThreadName.CONFIG_NODE_PROCEDURE_WORKER.getName());
+ this.timeoutExecutor =
+ new TimeoutExecutorThread<>(
+ this, threadGroup, ThreadName.CONFIG_NODE_TIMEOUT_EXECUTOR.getName());
+ this.workerMonitorExecutor =
+ new TimeoutExecutorThread<>(
+ this, threadGroup, ThreadName.CONFIG_NODE_WORKER_THREAD_MONITOR.getName());
+ workId.set(0);
+ workerThreads = new CopyOnWriteArrayList<>();
+ for (int i = 0; i < corePoolSize; i++) {
+ workerThreads.add(new WorkerThread(threadGroup));
+ }
+ // Add worker monitor
+ workerMonitorExecutor.add(new WorkerMonitor());
+
+ scheduler.start();
+ recover();
+ }
+
+ private void recover() {
+ // 1.Build rollback stack
+ List<Procedure<Env>> procedureList = getProcedureListFromDifferentVersion();
+ // Load procedure wal file
+ for (Procedure<Env> proc : procedureList) {
+ if (proc.isFinished()) {
+ completed.putIfAbsent(proc.getProcId(), new CompletedProcedureContainer<>(proc));
+ } else {
+ if (!proc.hasParent()) {
+ rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
+ }
+ procedures.putIfAbsent(proc.getProcId(), proc);
+ }
+ }
+ List<Procedure<Env>> runnableList = new ArrayList<>();
+ List<Procedure<Env>> failedList = new ArrayList<>();
+ List<Procedure<Env>> waitingList = new ArrayList<>();
+ List<Procedure<Env>> waitingTimeoutList = new ArrayList<>();
+ for (Procedure<Env> proc : procedureList) {
+ if (proc.isFinished() && !proc.hasParent()) {
+ continue;
+ }
+ long rootProcedureId = getRootProcedureId(proc);
+ if (proc.hasParent()) {
+ Procedure<Env> parent = procedures.get(proc.getParentProcId());
+ if (parent != null && !proc.isFinished()) {
+ parent.incChildrenLatch();
+ }
+ }
+ RootProcedureStack<Env> rootStack = rollbackStack.get(rootProcedureId);
+ if (rootStack != null) {
+ rootStack.loadStack(proc);
+ }
+ proc.setRootProcedureId(rootProcedureId);
+ switch (proc.getState()) {
+ case RUNNABLE:
+ runnableList.add(proc);
+ break;
+ case FAILED:
+ failedList.add(proc);
+ break;
+ case WAITING:
+ waitingList.add(proc);
+ break;
+ case WAITING_TIMEOUT:
+ waitingTimeoutList.add(proc);
+ break;
+ case ROLLEDBACK:
+ case INITIALIZING:
+ LOG.error("Unexpected state:{} for {}", proc.getState(), proc);
+ throw new UnsupportedOperationException("Unexpected state");
+ default:
+ break;
+ }
+ }
+
+ waitingList.forEach(
+ procedure -> {
+ if (!procedure.hasChildren()) {
+ // Normally, WAITING procedures should be wakened by its children.
+ // But, there is a case that, all the children are successful, and before
+ // they can wake up their parent procedure, the master was killed.
+ // So, during recovering the procedures from ProcedureWal, its children
+ // are not loaded because of their SUCCESS state.
+ // So we need to continue to run this WAITING procedure. Before
+ // executing, we need to set its state to RUNNABLE.
+ procedure.setState(ProcedureState.RUNNABLE);
+ runnableList.add(procedure);
+ }
+ });
+ restoreLocks();
+
+ waitingTimeoutList.forEach(timeoutExecutor::add);
+
+ failedList.forEach(scheduler::addBack);
+ runnableList.forEach(scheduler::addBack);
+ scheduler.signalAll();
+ }
+
+ private List<Procedure<Env>> getProcedureListFromDifferentVersion() {
+ if (store.isOldVersionProcedureStore()) {
+ LOG.info("Old procedure directory detected, upgrade beginning...");
+ return store.load();
+ } else {
+ return store.getProcedures();
+ }
+ }
+
+ /**
+ * Helper to look up the root Procedure ID.
+ *
+ * @param proc given a specified procedure.
+ */
+ Long getRootProcedureId(Procedure<Env> proc) {
+ while (proc.hasParent()) {
+ proc = procedures.get(proc.getParentProcId());
+ if (proc == null) {
+ return NO_PROC_ID;
+ }
+ }
+ return proc.getProcId();
+ }
+
+ private void releaseLock(Procedure<Env> procedure, boolean force) {
+ if (force || !procedure.holdLock(this.environment) || procedure.isFinished()) {
+ procedure.doReleaseLock(this.environment, store);
+ }
+ }
+
+ private void restoreLock(Procedure procedure, Set<Long> restored) {
+ procedure.restoreLock(environment);
+ restored.add(procedure.getProcId());
+ }
+
+ private void restoreLocks(Deque<Procedure<Env>> stack, Set<Long> restored) {
+ while (!stack.isEmpty()) {
+ restoreLock(stack.pop(), restored);
+ }
+ }
+
+ private void restoreLocks() {
+ Set<Long> restored = new HashSet<>();
+ Deque<Procedure<Env>> stack = new ArrayDeque<>();
+ procedures
+ .values()
+ .forEach(
+ procedure -> {
+ while (procedure != null) {
+ if (restored.contains(procedure.getProcId())) {
+ restoreLocks(stack, restored);
+ return;
+ }
+ if (!procedure.hasParent()) {
+ restoreLock(procedure, restored);
+ restoreLocks(stack, restored);
+ return;
+ }
+ stack.push(procedure);
+ procedure = procedures.get(procedure.getParentProcId());
+ }
+ });
+ }
+
+ public void startWorkers() {
+ if (!running.compareAndSet(false, true)) {
+ LOG.warn("Already running");
+ return;
+ }
+ timeoutExecutor.start();
+ workerMonitorExecutor.start();
+ for (WorkerThread workerThread : workerThreads) {
+ workerThread.start();
+ }
+ LOG.info("{} procedure workers are started.", workerThreads.size());
+ }
+
+ public void startCompletedCleaner(long cleanTimeInterval, long cleanEvictTTL) {
+ addInternalProcedure(
+ new CompletedProcedureRecycler(store, completed, cleanTimeInterval, cleanEvictTTL));
+ }
+
+ public void addInternalProcedure(InternalProcedure interalProcedure) {
+ if (interalProcedure == null) {
+ return;
+ }
+ interalProcedure.setState(ProcedureState.WAITING_TIMEOUT);
+ timeoutExecutor.add(interalProcedure);
+ }
+
+ public boolean removeInternalProcedure(InternalProcedure internalProcedure) {
+ if (internalProcedure == null) {
+ return true;
+ }
+ internalProcedure.setState(ProcedureState.SUCCESS);
+ return timeoutExecutor.remove(internalProcedure);
+ }
+
+ /**
+ * Executes procedure
+ *
+ * <p>Calls doExecute() if success and return subprocedures submit sub procs set the state to
+ * WAITING, wait for all sub procs completed. else if no sub procs procedure completed
+ * successfully set procedure's parent to RUNNABLE in case of failure start rollback of the
+ * procedure.
+ *
+ * @param proc procedure
+ */
+ private void executeProcedure(Procedure<Env> proc) {
+ if (proc.isFinished()) {
+ LOG.debug("{} is already finished.", proc);
+ return;
+ }
+ final Long rootProcId = getRootProcedureId(proc);
+ if (rootProcId == null) {
+ LOG.warn("Rollback because parent is done/rolledback, proc is {}", proc);
+ executeRollback(proc);
+ return;
+ }
+ RootProcedureStack<Env> rootProcStack = rollbackStack.get(rootProcId);
+ if (rootProcStack == null) {
+ LOG.warn("Rollback stack is null for {}", proc.getProcId());
+ return;
+ }
+ ProcedureLockState lockState = null;
+ try {
+ do {
+ if (!rootProcStack.acquire()) {
+ if (rootProcStack.setRollback()) {
+ lockState = executeRootStackRollback(rootProcId, rootProcStack);
+ switch (lockState) {
+ case LOCK_ACQUIRED:
+ break;
+ case LOCK_EVENT_WAIT:
+ LOG.info("LOCK_EVENT_WAIT rollback {}", proc);
+ rootProcStack.unsetRollback();
+ break;
+ case LOCK_YIELD_WAIT:
+ rootProcStack.unsetRollback();
+ scheduler.yield(proc);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ } else {
+ if (!proc.wasExecuted()) {
+ switch (executeRollback(proc)) {
+ case LOCK_ACQUIRED:
+ break;
+ case LOCK_EVENT_WAIT:
+ LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc);
+ break;
+ case LOCK_YIELD_WAIT:
+ scheduler.yield(proc);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+ }
+ break;
+ }
+ lockState = acquireLock(proc);
+ switch (lockState) {
+ case LOCK_ACQUIRED:
+ executeProcedure(rootProcStack, proc);
+ break;
+ case LOCK_YIELD_WAIT:
+ case LOCK_EVENT_WAIT:
+ LOG.info("{} lockstate is {}", proc, lockState);
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ rootProcStack.release();
+
+ if (proc.isSuccess()) {
+ // update metrics on finishing the procedure
+ proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);
+ LOG.debug("{} finished in {}ms successfully.", proc, proc.elapsedTime());
+ if (proc.getProcId() == rootProcId) {
+ rootProcedureCleanup(proc);
+ } else {
+ executeCompletionCleanup(proc);
+ }
+ return;
+ }
+
+ } while (rootProcStack.isFailed());
+ } finally {
+ // Only after procedure has completed execution can it be allowed to be rescheduled to prevent
+ // data races
+ if (Objects.equals(lockState, ProcedureLockState.LOCK_EVENT_WAIT)) {
+ LOG.info("procedureId {} wait for lock.", proc.getProcId());
+ ((ConfigNodeProcedureEnv) this.environment).getNodeLock().waitProcedure(proc);
+ }
+ }
+ }
+
+ /**
+ * execute procedure and submit its children
+ *
+ * @param rootProcStack procedure's root proc stack
+ * @param proc procedure
+ */
+ private void executeProcedure(RootProcedureStack rootProcStack, Procedure<Env> proc) {
+ if (proc.getState() != ProcedureState.RUNNABLE) {
+ LOG.error(
+ "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}", proc);
+ return;
+ }
+ boolean reExecute;
+
+ Procedure<Env>[] subprocs = null;
+ do {
+ reExecute = false;
+ try {
+ subprocs = proc.doExecute(this.environment);
+ if (subprocs != null && subprocs.length == 0) {
+ subprocs = null;
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupt during execution, suspend or retry it later.", e);
+ yieldProcedure(proc);
+ } catch (Throwable e) {
+ LOG.error("CODE-BUG:{}", proc, e);
+ proc.setFailure(new ProcedureException(e.getMessage(), e));
+ }
+
+ if (!proc.isFailed()) {
+ if (subprocs != null) {
+ if (subprocs.length == 1 && subprocs[0] == proc) {
+ subprocs = null;
+ reExecute = true;
+ } else {
+ subprocs = initializeChildren(rootProcStack, proc, subprocs);
+ LOG.info("Initialized sub procs:{}", Arrays.toString(subprocs));
+ }
+ } else if (proc.getState() == ProcedureState.WAITING_TIMEOUT) {
+ LOG.info("Added into timeoutExecutor {}", proc);
+ } else {
+ proc.setState(ProcedureState.SUCCESS);
+ }
+ }
+ // add procedure into rollback stack.
+ rootProcStack.addRollbackStep(proc);
+
+ updateStoreOnExecution(rootProcStack, proc, subprocs);
+
+ if (!store.isRunning()) {
+ return;
+ }
+
+ if (proc.isRunnable() && proc.isYieldAfterExecution(this.environment)) {
+ yieldProcedure(proc);
+ return;
+ }
+ } while (reExecute);
+
+ if (subprocs != null && !proc.isFailed()) {
+ submitChildrenProcedures(subprocs);
+ }
+
+ releaseLock(proc, false);
+ if (proc.isFinished() && proc.hasParent()) {
+ countDownChildren(rootProcStack, proc);
+ }
+ }
+
+ /**
+ * Serve as a countdown latch to check whether all children have already completed.
+ *
+ * @param rootProcStack root procedure stack
+ * @param proc proc
+ */
+ private void countDownChildren(RootProcedureStack rootProcStack, Procedure<Env> proc) {
+ Procedure<Env> parent = procedures.get(proc.getParentProcId());
+ if (parent == null && rootProcStack.isRollingback()) {
+ return;
+ }
+ if (parent != null && parent.tryRunnable()) {
+ // If success, means all its children have completed, move parent to front of the queue.
+ store.update(parent);
+ scheduler.addFront(parent);
+ LOG.info(
+ "Finished subprocedure pid={}, resume processing ppid={}",
+ proc.getProcId(),
+ parent.getProcId());
+ }
+ }
+
+ /**
+ * Submit children procedures.
+ *
+ * @param subprocs children procedures
+ */
+ private void submitChildrenProcedures(Procedure<Env>[] subprocs) {
+ for (Procedure<Env> subproc : subprocs) {
+ subproc.updateMetricsOnSubmit(getEnvironment());
+ procedures.put(subproc.getProcId(), subproc);
+ scheduler.addFront(subproc);
+ LOG.info("Sub-Procedure pid={} has been submitted", subproc.getProcId());
+ }
+ }
+
+ private void updateStoreOnExecution(
+ RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
+ if (subprocs != null && !proc.isFailed()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stored {}, children {}", proc, Arrays.toString(subprocs));
+ }
+ store.update(subprocs);
+ } else {
+ LOG.debug("Store update {}", proc);
+ if (proc.isFinished() && !proc.hasParent()) {
+ final long[] childProcIds = rootProcStack.getSubprocedureIds();
+ if (childProcIds != null) {
+ store.delete(childProcIds);
+ for (long childProcId : childProcIds) {
+ procedures.remove(childProcId);
+ }
+ } else {
+ store.update(proc);
+ }
+ } else {
+ store.update(proc);
+ }
+ }
+ }
+
+ private Procedure<Env>[] initializeChildren(
+ RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
+ final long rootProcedureId = getRootProcedureId(proc);
+ for (int i = 0; i < subprocs.length; i++) {
+ Procedure<Env> subproc = subprocs[i];
+ if (subproc == null) {
+ String errMsg = "subproc[" + i + "] is null, aborting procedure";
+ proc.setFailure(new ProcedureException((errMsg), new IllegalArgumentException(errMsg)));
+ return null;
+ }
+ subproc.setParentProcId(proc.getProcId());
+ subproc.setRootProcId(rootProcedureId);
+ subproc.setProcId(store.getNextProcId());
+ subproc.setProcRunnable();
+ rootProcStack.addSubProcedure(subproc);
+ }
+
+ if (!proc.isFailed()) {
+ proc.setChildrenLatch(subprocs.length);
+ switch (proc.getState()) {
+ case RUNNABLE:
+ proc.setState(ProcedureState.WAITING);
+ break;
+ case WAITING_TIMEOUT:
+ timeoutExecutor.add(proc);
+ break;
+ default:
+ break;
+ }
+ }
+ return subprocs;
+ }
+
+ private void yieldProcedure(Procedure<Env> proc) {
+ releaseLock(proc, false);
+ scheduler.yield(proc);
+ }
+
+ /**
+ * Rollback full root procedure stack.
+ *
+ * @param rootProcId root procedure id
+ * @param procedureStack root procedure stack
+ * @return lock state
+ */
+ private ProcedureLockState executeRootStackRollback(
+ Long rootProcId, RootProcedureStack procedureStack) {
+ Procedure<Env> rootProcedure = procedures.get(rootProcId);
+ ProcedureException exception = rootProcedure.getException();
+ if (exception == null) {
+ exception = procedureStack.getException();
+ rootProcedure.setFailure(exception);
+ store.update(rootProcedure);
+ }
+ List<Procedure<Env>> subprocStack = procedureStack.getSubproceduresStack();
+ int stackTail = subprocStack.size();
+ while (stackTail-- > 0) {
+ Procedure<Env> procedure = subprocStack.get(stackTail);
+ if (procedure.isSuccess()) {
+ subprocStack.remove(stackTail);
+ cleanupAfterRollback(procedure);
+ continue;
+ }
+ ProcedureLockState lockState = acquireLock(procedure);
+ if (lockState != ProcedureLockState.LOCK_ACQUIRED) {
+ return lockState;
+ }
+ lockState = executeRollback(procedure);
+ releaseLock(procedure, false);
+
+ boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED;
+ abortRollback |= !isRunning() || !store.isRunning();
+ if (abortRollback) {
+ return lockState;
+ }
+
+ if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) {
+ return ProcedureLockState.LOCK_YIELD_WAIT;
+ }
+
+ if (procedure != rootProcedure) {
+ executeCompletionCleanup(procedure);
+ }
+ }
+
+ LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime());
+ rootProcedureCleanup(rootProcedure);
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ private ProcedureLockState acquireLock(Procedure<Env> proc) {
+ if (proc.hasLock()) {
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+ return proc.doAcquireLock(this.environment, store);
+ }
+
+ /**
+ * do execute defined in procedure and then update store or remove completely in case it is a
+ * child.
+ *
+ * @param procedure procedure
+ * @return procedure lock state
+ */
+ private ProcedureLockState executeRollback(Procedure<Env> procedure) {
+ try {
+ procedure.doRollback(this.environment);
+ } catch (IOException e) {
+ LOG.error("Roll back failed for {}", procedure, e);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted exception occured for {}", procedure, e);
+ } catch (Throwable t) {
+ LOG.error("CODE-BUG: runtime exception for {}", procedure, t);
+ }
+ cleanupAfterRollback(procedure);
+ return ProcedureLockState.LOCK_ACQUIRED;
+ }
+
+ private void cleanupAfterRollback(Procedure<Env> procedure) {
+ if (procedure.removeStackIndex()) {
+ if (!procedure.isSuccess()) {
+ procedure.setState(ProcedureState.ROLLEDBACK);
+ }
+
+ // update metrics on finishing the procedure (fail)
+ procedure.updateMetricsOnFinish(getEnvironment(), procedure.elapsedTime(), false);
+
+ if (procedure.hasParent()) {
+ store.delete(procedure.getProcId());
+ procedures.remove(procedure.getProcId());
+ } else {
+ final long[] childProcIds = rollbackStack.get(procedure.getProcId()).getSubprocedureIds();
+ if (childProcIds != null) {
+ store.delete(childProcIds);
+ } else {
+ store.update(procedure);
+ }
+ }
+ } else {
+ store.update(procedure);
+ }
+ }
+
+ private void executeCompletionCleanup(Procedure<Env> proc) {
+ if (proc.hasLock()) {
+ releaseLock(proc, true);
+ }
+ }
+
+ private void rootProcedureCleanup(Procedure<Env> proc) {
+ executeCompletionCleanup(proc);
+ CompletedProcedureContainer<Env> retainer = new CompletedProcedureContainer<>(proc);
+ completed.put(proc.getProcId(), retainer);
+ rollbackStack.remove(proc.getProcId());
+ procedures.remove(proc.getProcId());
+ }
+
+ /**
+ * Add a Procedure to executor.
+ *
+ * @param procedure procedure
+ * @return procedure id
+ */
+ private long pushProcedure(Procedure<Env> procedure) {
+ final long currentProcId = procedure.getProcId();
+ // Update metrics on start of a procedure
+ procedure.updateMetricsOnSubmit(getEnvironment());
+ RootProcedureStack<Env> stack = new RootProcedureStack<>();
+ rollbackStack.put(currentProcId, stack);
+ procedures.put(currentProcId, procedure);
+ scheduler.addBack(procedure);
+ return procedure.getProcId();
+ }
+
+ private class WorkerThread extends StoppableThread {
+ private final AtomicLong startTime = new AtomicLong(Long.MAX_VALUE);
+ private final AtomicReference<Procedure<Env>> activeProcedure = new AtomicReference<>();
+ protected long keepAliveTime = -1;
+
+ public WorkerThread(ThreadGroup threadGroup) {
+ this(threadGroup, "ProcedureCoreWorker-");
+ }
+
+ public WorkerThread(ThreadGroup threadGroup, String prefix) {
+ super(threadGroup, prefix + workId.incrementAndGet());
+ setDaemon(true);
+ }
+
+ @Override
+ public void sendStopSignal() {
+ scheduler.signalAll();
+ }
+
+ @Override
+ public void run() {
+ long lastUpdated = System.currentTimeMillis();
+ try {
+ while (isRunning() && keepAlive(lastUpdated)) {
+ Procedure<Env> procedure = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);
+ if (procedure == null) {
+ Thread.sleep(1000);
+ continue;
+ }
+ this.activeProcedure.set(procedure);
+ activeExecutorCount.incrementAndGet();
+ startTime.set(System.currentTimeMillis());
+ executeProcedure(procedure);
+ activeExecutorCount.decrementAndGet();
+ LOG.trace(
+ "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get());
+ this.activeProcedure.set(null);
+ lastUpdated = System.currentTimeMillis();
+ startTime.set(lastUpdated);
+ }
+
+ } catch (Exception e) {
+ if (this.activeProcedure.get() != null) {
+ LOG.warn(
+ "Exception happened when worker {} execute procedure {}",
+ getName(),
+ this.activeProcedure.get(),
+ e);
+ }
+ } finally {
+ LOG.info("Procedure worker {} terminated.", getName());
+ }
+ workerThreads.remove(this);
+ }
+
+ protected boolean keepAlive(long lastUpdated) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ Procedure<?> p = this.activeProcedure.get();
+ return getName() + "(pid=" + (p == null ? NO_PROC_ID : p.getProcId() + ")");
+ }
+
+ /**
+ * @return the time since the current procedure is running
+ */
+ public long getCurrentRunTime() {
+ return System.currentTimeMillis() - startTime.get();
+ }
+ }
+
+ // A temporary worker thread will be launched when too many core workers are stuck.
+ // They will timeout after keepAliveTime if there is no procedure to run.
+ private final class TemporaryWorkerThread extends WorkerThread {
+
+ public TemporaryWorkerThread(ThreadGroup group) {
+ super(group, "ProcedureTemporaryWorker-");
+ this.keepAliveTime = TimeUnit.SECONDS.toMillis(10);
+ }
+
+ @Override
+ protected boolean keepAlive(long lastUpdate) {
+ return System.currentTimeMillis() - lastUpdate < keepAliveTime;
+ }
+ }
+
+ private final class WorkerMonitor extends InternalProcedure<Env> {
+ private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 30000; // 30sec
+
+ private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 60000; // 60sec
+
+ private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck
+
+ public WorkerMonitor() {
+ super(DEFAULT_WORKER_MONITOR_INTERVAL);
+ updateTimestamp();
+ }
+
+ private int calculateRunningAndStuckWorkers() {
+ // Check if any of the worker is stuck
+ int runningCount = 0, stuckCount = 0;
+ for (WorkerThread worker : workerThreads) {
+ final Procedure<?> proc = worker.activeProcedure.get();
+ if (proc == null) {
+ continue;
+ }
+ runningCount++;
+ // WARN the worker is stuck
+ if (worker.getCurrentRunTime() > DEFAULT_WORKER_STUCK_THRESHOLD) {
+ stuckCount++;
+ LOG.warn(
+ "Worker stuck {}({}), run time {} ms",
+ worker,
+ proc.getProcType(),
+ worker.getCurrentRunTime());
+ }
+ LOG.info(
+ "Procedure workers: {} is running, {} is running and stuck", runningCount, stuckCount);
+ }
+ return stuckCount;
+ }
+
+ private void checkThreadCount(final int stuckCount) {
+ // Nothing to do if there are no runnable tasks
+ if (stuckCount < 1 || !scheduler.hasRunnables()) {
+ return;
+ }
+ // Add a new thread if the worker stuck percentage exceed the threshold limit
+ // and every handler is active.
+ final float stuckPerc = ((float) stuckCount) / workerThreads.size();
+ // Let's add new worker thread more aggressively, as they will timeout finally if there is no
+ // work to do.
+ if (stuckPerc >= DEFAULT_WORKER_ADD_STUCK_PERCENTAGE && workerThreads.size() < maxPoolSize) {
+ final TemporaryWorkerThread worker = new TemporaryWorkerThread(threadGroup);
+ workerThreads.add(worker);
+ worker.start();
+ LOG.debug("Added new worker thread {}", worker);
+ }
+ }
+
+ @Override
+ protected void periodicExecute(Env env) {
+ final int stuckCount = calculateRunningAndStuckWorkers();
+ checkThreadCount(stuckCount);
+ updateTimestamp();
+ }
+ }
+
+ public int getWorkerThreadCount() {
+ return workerThreads.size();
+ }
+
+ public long getActiveWorkerThreadCount() {
+ return workerThreads.stream().filter(worker -> worker.activeProcedure.get() != null).count();
+ }
+
+ public boolean isRunning() {
+ return running.get();
+ }
+
+ public void stop() {
+ if (!running.getAndSet(false)) {
+ return;
+ }
+ LOG.info("Stopping");
+ scheduler.stop();
+ timeoutExecutor.sendStopSignal();
+ }
+
+ public void join() {
+ timeoutExecutor.awaitTermination();
+ workerMonitorExecutor.awaitTermination();
+ for (WorkerThread workerThread : workerThreads) {
+ workerThread.awaitTermination();
+ }
+ try {
+ threadGroup.destroy();
+ } catch (IllegalThreadStateException e) {
+ LOG.warn(
+ "ProcedureExecutor threadGroup {} contains running threads which are used by non-procedure module.",
+ this.threadGroup);
+ this.threadGroup.list();
+ }
+ }
+
+ public boolean isStarted(long procId) {
+ Procedure<Env> procedure = procedures.get(procId);
+ if (procedure == null) {
+ return completed.get(procId) != null;
+ }
+ return procedure.wasExecuted();
+ }
+
+ public boolean isFinished(final long procId) {
+ return !procedures.containsKey(procId);
+ }
+
+ public ConcurrentHashMap<Long, Procedure<Env>> getProcedures() {
+ return procedures;
+ }
+
+ // -----------------------------CLIENT IMPLEMENTATION-----------------------------------
+ /**
+ * Submit a new root-procedure to the executor, called by client.
+ *
+ * @param procedure root procedure
+ * @return procedure id
+ */
+ public long submitProcedure(Procedure<Env> procedure) {
+ Preconditions.checkArgument(procedure.getState() == ProcedureState.INITIALIZING);
+ Preconditions.checkArgument(!procedure.hasParent(), "Unexpected parent", procedure);
+ // Initialize the procedure
+ procedure.setProcId(store.getNextProcId());
+ procedure.setProcRunnable();
+ // Commit the transaction
+ store.update(procedure);
+ LOG.debug("{} is stored.", procedure);
+ // Add the procedure to the executor
+ return pushProcedure(procedure);
+ }
+
+ public ProcedureScheduler getScheduler() {
+ return scheduler;
+ }
+
+ public Env getEnvironment() {
+ return environment;
+ }
+
+ public IProcedureStore getStore() {
+ return store;
+ }
+
+ public RootProcedureStack<Env> getRollbackStack(long rootProcId) {
+ return rollbackStack.get(rootProcId);
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
index 8ac6ff8f99b..25533667431 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
@@ -1,185 +1,184 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.commons.service.metric.enums.Metric;
-1import org.apache.iotdb.commons.service.metric.enums.Tag;
-1import org.apache.iotdb.confignode.manager.ProcedureManager;
-1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-1import org.apache.iotdb.metrics.AbstractMetricService;
-1import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
-1import org.apache.iotdb.metrics.metricsets.IMetricSet;
-1import org.apache.iotdb.metrics.type.Counter;
-1import org.apache.iotdb.metrics.type.Timer;
-1import org.apache.iotdb.metrics.utils.MetricLevel;
-1import org.apache.iotdb.metrics.utils.MetricType;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.util.Map;
-1import java.util.Optional;
-1import java.util.concurrent.ConcurrentHashMap;
-1
-1public class ProcedureMetrics implements IMetricSet {
-1 private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureMetrics.class);
-1
-1 private final ProcedureManager procedureManager;
-1
-1 private final ProcedureExecutor<ConfigNodeProcedureEnv> executor;
-1
-1 private final ProcedureScheduler scheduler;
-1
-1 private final Map<String, ProcedureMetricItems> metricItemsMap = new ConcurrentHashMap<>();
-1
-1 private AbstractMetricService metricService;
-1
-1 public ProcedureMetrics(ProcedureManager procedureManager) {
-1 this.procedureManager = procedureManager;
-1 this.executor = procedureManager.getExecutor();
-1 this.scheduler = procedureManager.getScheduler();
-1 }
-1
-1 @Override
-1 public void bindTo(AbstractMetricService metricService) {
-1 this.metricService = metricService;
-1 bindThreadMetrics(metricService);
-1 }
-1
-1 @Override
-1 public void unbindFrom(AbstractMetricService metricService) {
-1 unbindThreadMetrics(metricService);
-1
-1 for (ProcedureMetricItems items : metricItemsMap.values()) {
-1 items.unbindMetricItems(metricService);
-1 }
-1 metricItemsMap.clear();
-1 }
-1
-1 private void bindThreadMetrics(AbstractMetricService metricService) {
-1 metricService.createAutoGauge(
-1 Metric.PROCEDURE_WORKER_THREAD_COUNT.toString(),
-1 MetricLevel.CORE,
-1 executor,
-1 ProcedureExecutor::getWorkerThreadCount);
-1 metricService.createAutoGauge(
-1 Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString(),
-1 MetricLevel.CORE,
-1 executor,
-1 ProcedureExecutor::getActiveWorkerThreadCount);
-1 metricService.createAutoGauge(
-1 Metric.PROCEDURE_QUEUE_LENGTH.toString(),
-1 MetricLevel.CORE,
-1 scheduler,
-1 ProcedureScheduler::size);
-1 }
-1
-1 private void unbindThreadMetrics(AbstractMetricService metricService) {
-1 metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_WORKER_THREAD_COUNT.toString());
-1 metricService.remove(
-1 MetricType.AUTO_GAUGE, Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString());
-1 metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_QUEUE_LENGTH.toString());
-1 }
-1
-1 public void updateMetricsOnSubmit(String procType) {
-1 Optional.ofNullable(metricService)
-1 .ifPresent(
-1 service -> {
-1 metricItemsMap
-1 .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
-1 .updateMetricsOnSubmit();
-1 });
-1 }
-1
-1 public void updateMetricsOnFinish(String procType, long runtime, boolean success) {
-1 Optional.ofNullable(metricService)
-1 .ifPresent(
-1 service -> {
-1 metricItemsMap
-1 .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
-1 .updateMetricsOnFinish(runtime, success);
-1 });
-1 }
-1
-1 class ProcedureMetricItems {
-1 private final String procType;
-1 private Counter submittedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
-1 private Counter failedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
-1 private Timer executionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
-1
-1 public ProcedureMetricItems(String procType, AbstractMetricService metricService) {
-1 this.procType = procType;
-1 bindMetricItems(metricService);
-1 }
-1
-1 public void bindMetricItems(AbstractMetricService metricService) {
-1 this.submittedCounter =
-1 metricService.getOrCreateCounter(
-1 Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
-1 MetricLevel.CORE,
-1 Tag.TYPE.toString(),
-1 procType);
-1 this.failedCounter =
-1 metricService.getOrCreateCounter(
-1 Metric.PROCEDURE_FAILED_COUNT.toString(),
-1 MetricLevel.CORE,
-1 Tag.TYPE.toString(),
-1 procType);
-1 this.executionTimer =
-1 metricService.getOrCreateTimer(
-1 Metric.PROCEDURE_EXECUTION_TIME.toString(),
-1 MetricLevel.CORE,
-1 Tag.TYPE.toString(),
-1 procType);
-1 }
-1
-1 public void unbindMetricItems(AbstractMetricService metricService) {
-1 metricService.remove(
-1 MetricType.COUNTER,
-1 Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
-1 Tag.TYPE.toString(),
-1 procType);
-1 metricService.remove(
-1 MetricType.COUNTER,
-1 Metric.PROCEDURE_FAILED_COUNT.toString(),
-1 Tag.TYPE.toString(),
-1 procType);
-1 metricService.remove(
-1 MetricType.TIMER,
-1 Metric.PROCEDURE_EXECUTION_TIME.toString(),
-1 Tag.TYPE.toString(),
-1 procType);
-1 }
-1
-1 public void updateMetricsOnSubmit() {
-1 submittedCounter.inc();
-1 }
-1
-1 public void updateMetricsOnFinish(long runtime, boolean success) {
-1 if (success) {
-1 executionTimer.updateMillis(runtime);
-1 } else {
-1 failedCounter.inc();
-1 }
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.commons.service.metric.enums.Tag;
+import org.apache.iotdb.confignode.manager.ProcedureManager;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Counter;
+import org.apache.iotdb.metrics.type.Timer;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+
+public class ProcedureMetrics implements IMetricSet {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureMetrics.class);
+
+ private final ProcedureManager procedureManager;
+
+ private final ProcedureExecutor<ConfigNodeProcedureEnv> executor;
+
+ private final ProcedureScheduler scheduler;
+
+ private final Map<String, ProcedureMetricItems> metricItemsMap = new ConcurrentHashMap<>();
+
+ private AbstractMetricService metricService;
+
+ public ProcedureMetrics(ProcedureManager procedureManager) {
+ this.procedureManager = procedureManager;
+ this.executor = procedureManager.getExecutor();
+ this.scheduler = procedureManager.getScheduler();
+ }
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ this.metricService = metricService;
+ bindThreadMetrics(metricService);
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ unbindThreadMetrics(metricService);
+
+ for (ProcedureMetricItems items : metricItemsMap.values()) {
+ items.unbindMetricItems(metricService);
+ }
+ metricItemsMap.clear();
+ }
+
+ private void bindThreadMetrics(AbstractMetricService metricService) {
+ metricService.createAutoGauge(
+ Metric.PROCEDURE_WORKER_THREAD_COUNT.toString(),
+ MetricLevel.CORE,
+ executor,
+ ProcedureExecutor::getWorkerThreadCount);
+ metricService.createAutoGauge(
+ Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString(),
+ MetricLevel.CORE,
+ executor,
+ ProcedureExecutor::getActiveWorkerThreadCount);
+ metricService.createAutoGauge(
+ Metric.PROCEDURE_QUEUE_LENGTH.toString(),
+ MetricLevel.CORE,
+ scheduler,
+ ProcedureScheduler::size);
+ }
+
+ private void unbindThreadMetrics(AbstractMetricService metricService) {
+ metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_WORKER_THREAD_COUNT.toString());
+ metricService.remove(
+ MetricType.AUTO_GAUGE, Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString());
+ metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_QUEUE_LENGTH.toString());
+ }
+
+ public void updateMetricsOnSubmit(String procType) {
+ Optional.ofNullable(metricService)
+ .ifPresent(
+ service -> {
+ metricItemsMap
+ .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
+ .updateMetricsOnSubmit();
+ });
+ }
+
+ public void updateMetricsOnFinish(String procType, long runtime, boolean success) {
+ Optional.ofNullable(metricService)
+ .ifPresent(
+ service -> {
+ metricItemsMap
+ .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
+ .updateMetricsOnFinish(runtime, success);
+ });
+ }
+
+ class ProcedureMetricItems {
+ private final String procType;
+ private Counter submittedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Counter failedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ private Timer executionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+
+ public ProcedureMetricItems(String procType, AbstractMetricService metricService) {
+ this.procType = procType;
+ bindMetricItems(metricService);
+ }
+
+ public void bindMetricItems(AbstractMetricService metricService) {
+ this.submittedCounter =
+ metricService.getOrCreateCounter(
+ Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
+ MetricLevel.CORE,
+ Tag.TYPE.toString(),
+ procType);
+ this.failedCounter =
+ metricService.getOrCreateCounter(
+ Metric.PROCEDURE_FAILED_COUNT.toString(),
+ MetricLevel.CORE,
+ Tag.TYPE.toString(),
+ procType);
+ this.executionTimer =
+ metricService.getOrCreateTimer(
+ Metric.PROCEDURE_EXECUTION_TIME.toString(),
+ MetricLevel.CORE,
+ Tag.TYPE.toString(),
+ procType);
+ }
+
+ public void unbindMetricItems(AbstractMetricService metricService) {
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
+ Tag.TYPE.toString(),
+ procType);
+ metricService.remove(
+ MetricType.COUNTER,
+ Metric.PROCEDURE_FAILED_COUNT.toString(),
+ Tag.TYPE.toString(),
+ procType);
+ metricService.remove(
+ MetricType.TIMER,
+ Metric.PROCEDURE_EXECUTION_TIME.toString(),
+ Tag.TYPE.toString(),
+ procType);
+ }
+
+ public void updateMetricsOnSubmit() {
+ submittedCounter.inc();
+ }
+
+ public void updateMetricsOnFinish(long runtime, boolean success) {
+ if (success) {
+ executionTimer.updateMillis(runtime);
+ } else {
+ failedCounter.inc();
+ }
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
index 88caab664cd..ff19872a550 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
@@ -1,178 +1,177 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.util.ArrayList;
-1import java.util.HashSet;
-1import java.util.List;
-1import java.util.Set;
-1
-1public class RootProcedureStack<Env> {
-1 private static final Logger LOG = LoggerFactory.getLogger(RootProcedureStack.class);
-1
-1 private enum State {
-1 RUNNING, // The Procedure is running or ready to run
-1 FAILED, // The Procedure failed, waiting for the rollback executing
-1 ROLLINGBACK, // The Procedure failed and the execution was rolledback
-1 }
-1
-1 private Set<Procedure<Env>> subprocs = null;
-1 private ArrayList<Procedure<Env>> subprocStack = null;
-1 private State state = State.RUNNING;
-1 private int running = 0;
-1
-1 public synchronized boolean isFailed() {
-1 switch (state) {
-1 case ROLLINGBACK:
-1 case FAILED:
-1 return true;
-1 default:
-1 break;
-1 }
-1 return false;
-1 }
-1
-1 public synchronized boolean isRollingback() {
-1 return state == State.ROLLINGBACK;
-1 }
-1
-1 /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
-1 protected synchronized boolean setRollback() {
-1 if (running == 0 && state == State.FAILED) {
-1 state = State.ROLLINGBACK;
-1 return true;
-1 }
-1 return false;
-1 }
-1
-1 /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
-1 protected synchronized void unsetRollback() {
-1 assert state == State.ROLLINGBACK;
-1 state = State.FAILED;
-1 }
-1
-1 protected synchronized long[] getSubprocedureIds() {
-1 if (subprocs == null) {
-1 return new long[0];
-1 }
-1 return subprocs.stream().mapToLong(Procedure::getProcId).toArray();
-1 }
-1
-1 protected synchronized List<Procedure<Env>> getSubproceduresStack() {
-1 return subprocStack;
-1 }
-1
-1 protected synchronized ProcedureException getException() {
-1 if (subprocStack != null) {
-1 for (Procedure<Env> proc : subprocStack) {
-1 if (proc.hasException()) {
-1 return proc.getException();
-1 }
-1 }
-1 }
-1 return null;
-1 }
-1
-1 /** Called by the {@link ProcedureExecutor} to mark the procedure step as running. */
-1 protected synchronized boolean acquire() {
-1 if (state != State.RUNNING) {
-1 return false;
-1 }
-1
-1 running++;
-1 return true;
-1 }
-1
-1 /** Called by the {@link ProcedureExecutor} to mark the procedure step as finished. */
-1 protected synchronized void release() {
-1 running--;
-1 }
-1
-1 protected synchronized void abort() {
-1 if (state == State.RUNNING) {
-1 state = State.FAILED;
-1 }
-1 }
-1
-1 /**
-1 * Called by the {@link ProcedureExecutor} after the procedure step is completed, to add the step
-1 * to the rollback list (or procedure stack).
-1 */
-1 protected synchronized void addRollbackStep(Procedure<Env> proc) {
-1 if (proc.isFailed()) {
-1 state = State.FAILED;
-1 }
-1 if (subprocStack == null) {
-1 subprocStack = new ArrayList<>();
-1 }
-1 proc.addStackIndex(subprocStack.size());
-1 LOG.trace("Add procedure {} as the {}th rollback step", proc, subprocStack.size());
-1 subprocStack.add(proc);
-1 }
-1
-1 protected synchronized void addSubProcedure(Procedure<Env> proc) {
-1 if (!proc.hasParent()) {
-1 return;
-1 }
-1 if (subprocs == null) {
-1 subprocs = new HashSet<>();
-1 }
-1 subprocs.add(proc);
-1 }
-1
-1 /**
-1 * Called on store load by the {@link ProcedureExecutor} to load part of the stack.
-1 *
-1 * <p>Each procedure has its own stack-positions. Which means we have to write to the store only
-1 * the Procedure we executed, and nothing else. On load we recreate the full stack by aggregating
-1 * each procedure stack-positions.
-1 */
-1 protected synchronized void loadStack(Procedure<Env> proc) {
-1 addSubProcedure(proc);
-1 int[] stackIndexes = proc.getStackIndexes();
-1 if (stackIndexes != null) {
-1 if (subprocStack == null) {
-1 subprocStack = new ArrayList<>();
-1 }
-1 int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();
-1 if (diff > 0) {
-1 subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
-1 while (diff-- > 0) {
-1 subprocStack.add(null);
-1 }
-1 }
-1 for (int stackIndex : stackIndexes) {
-1 subprocStack.set(stackIndex, proc);
-1 }
-1 }
-1 if (proc.getState() == ProcedureState.ROLLEDBACK) {
-1 state = State.ROLLINGBACK;
-1 } else if (proc.isFailed()) {
-1 state = State.FAILED;
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+public class RootProcedureStack<Env> {
+ private static final Logger LOG = LoggerFactory.getLogger(RootProcedureStack.class);
+
+ private enum State {
+ RUNNING, // The Procedure is running or ready to run
+ FAILED, // The Procedure failed, waiting for the rollback executing
+ ROLLINGBACK, // The Procedure failed and the execution was rolledback
+ }
+
+ private Set<Procedure<Env>> subprocs = null;
+ private ArrayList<Procedure<Env>> subprocStack = null;
+ private State state = State.RUNNING;
+ private int running = 0;
+
+ public synchronized boolean isFailed() {
+ switch (state) {
+ case ROLLINGBACK:
+ case FAILED:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ public synchronized boolean isRollingback() {
+ return state == State.ROLLINGBACK;
+ }
+
+ /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
+ protected synchronized boolean setRollback() {
+ if (running == 0 && state == State.FAILED) {
+ state = State.ROLLINGBACK;
+ return true;
+ }
+ return false;
+ }
+
+ /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
+ protected synchronized void unsetRollback() {
+ assert state == State.ROLLINGBACK;
+ state = State.FAILED;
+ }
+
+ protected synchronized long[] getSubprocedureIds() {
+ if (subprocs == null) {
+ return new long[0];
+ }
+ return subprocs.stream().mapToLong(Procedure::getProcId).toArray();
+ }
+
+ protected synchronized List<Procedure<Env>> getSubproceduresStack() {
+ return subprocStack;
+ }
+
+ protected synchronized ProcedureException getException() {
+ if (subprocStack != null) {
+ for (Procedure<Env> proc : subprocStack) {
+ if (proc.hasException()) {
+ return proc.getException();
+ }
+ }
+ }
+ return null;
+ }
+
+ /** Called by the {@link ProcedureExecutor} to mark the procedure step as running. */
+ protected synchronized boolean acquire() {
+ if (state != State.RUNNING) {
+ return false;
+ }
+
+ running++;
+ return true;
+ }
+
+ /** Called by the {@link ProcedureExecutor} to mark the procedure step as finished. */
+ protected synchronized void release() {
+ running--;
+ }
+
+ protected synchronized void abort() {
+ if (state == State.RUNNING) {
+ state = State.FAILED;
+ }
+ }
+
+ /**
+ * Called by the {@link ProcedureExecutor} after the procedure step is completed, to add the step
+ * to the rollback list (or procedure stack).
+ */
+ protected synchronized void addRollbackStep(Procedure<Env> proc) {
+ if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ if (subprocStack == null) {
+ subprocStack = new ArrayList<>();
+ }
+ proc.addStackIndex(subprocStack.size());
+ LOG.trace("Add procedure {} as the {}th rollback step", proc, subprocStack.size());
+ subprocStack.add(proc);
+ }
+
+ protected synchronized void addSubProcedure(Procedure<Env> proc) {
+ if (!proc.hasParent()) {
+ return;
+ }
+ if (subprocs == null) {
+ subprocs = new HashSet<>();
+ }
+ subprocs.add(proc);
+ }
+
+ /**
+ * Called on store load by the {@link ProcedureExecutor} to load part of the stack.
+ *
+ * <p>Each procedure has its own stack-positions. Which means we have to write to the store only
+ * the Procedure we executed, and nothing else. On load we recreate the full stack by aggregating
+ * each procedure stack-positions.
+ */
+ protected synchronized void loadStack(Procedure<Env> proc) {
+ addSubProcedure(proc);
+ int[] stackIndexes = proc.getStackIndexes();
+ if (stackIndexes != null) {
+ if (subprocStack == null) {
+ subprocStack = new ArrayList<>();
+ }
+ int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();
+ if (diff > 0) {
+ subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
+ while (diff-- > 0) {
+ subprocStack.add(null);
+ }
+ }
+ for (int stackIndex : stackIndexes) {
+ subprocStack.set(stackIndex, proc);
+ }
+ }
+ if (proc.getState() == ProcedureState.ROLLEDBACK) {
+ state = State.ROLLINGBACK;
+ } else if (proc.isFailed()) {
+ state = State.FAILED;
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
index 5e6297d3d97..2e5bf8dd203 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
@@ -1,56 +1,55 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1public abstract class StoppableThread extends Thread {
-1
-1 private static final int JOIN_TIMEOUT = 250;
-1 private static final Logger LOG = LoggerFactory.getLogger(StoppableThread.class);
-1
-1 protected StoppableThread(ThreadGroup threadGroup, String name) {
-1 super(threadGroup, name);
-1 }
-1
-1 public abstract void sendStopSignal();
-1
-1 public void awaitTermination() {
-1 try {
-1 for (int i = 0; isAlive(); i++) {
-1 sendStopSignal();
-1 join(JOIN_TIMEOUT);
-1 if (i > 0 && (i % 8) == 0) {
-1 interrupt();
-1 }
-1 }
-1 } catch (InterruptedException e) {
-1 LOG.warn("{} join wait got interrupted", getName(), e);
-1 Thread.currentThread().interrupt();
-1 }
-1 }
-1
-1 @Override
-1 public void run() {
-1 // empty method
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class StoppableThread extends Thread {
+
+ private static final int JOIN_TIMEOUT = 250;
+ private static final Logger LOG = LoggerFactory.getLogger(StoppableThread.class);
+
+ protected StoppableThread(ThreadGroup threadGroup, String name) {
+ super(threadGroup, name);
+ }
+
+ public abstract void sendStopSignal();
+
+ public void awaitTermination() {
+ try {
+ for (int i = 0; isAlive(); i++) {
+ sendStopSignal();
+ join(JOIN_TIMEOUT);
+ if (i > 0 && (i % 8) == 0) {
+ interrupt();
+ }
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("{} join wait got interrupted", getName(), e);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ public void run() {
+ // empty method
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
index 1b028fa1288..5aaf9a623f5 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
@@ -1,108 +1,107 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure;
-1
-1import java.util.concurrent.DelayQueue;
-1import java.util.concurrent.Delayed;
-1import java.util.concurrent.TimeUnit;
-1
-1public class TimeoutExecutorThread<Env> extends StoppableThread {
-1
-1 private static final int DELAY_QUEUE_TIMEOUT = 20;
-1 private final ProcedureExecutor<Env> executor;
-1 private final DelayQueue<ProcedureDelayContainer<Env>> queue = new DelayQueue<>();
-1
-1 public TimeoutExecutorThread(
-1 ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, String name) {
-1 super(threadGroup, name);
-1 setDaemon(true);
-1 this.executor = envProcedureExecutor;
-1 }
-1
-1 public void add(Procedure<Env> procedure) {
-1 queue.add(new ProcedureDelayContainer<>(procedure));
-1 }
-1
-1 public boolean remove(Procedure<Env> procedure) {
-1 return queue.remove(new ProcedureDelayContainer<>(procedure));
-1 }
-1
-1 private ProcedureDelayContainer<Env> takeQuietly() {
-1 try {
-1 return queue.poll(DELAY_QUEUE_TIMEOUT, TimeUnit.SECONDS);
-1 } catch (InterruptedException e) {
-1 currentThread().interrupt();
-1 return null;
-1 }
-1 }
-1
-1 @Override
-1 public void run() {
-1 while (executor.isRunning()) {
-1 ProcedureDelayContainer<Env> delayTask = takeQuietly();
-1 if (delayTask == null) {
-1 continue;
-1 }
-1 Procedure<Env> procedure = delayTask.getProcedure();
-1 if (procedure instanceof InternalProcedure) {
-1 InternalProcedure internal = (InternalProcedure) procedure;
-1 internal.periodicExecute(executor.getEnvironment());
-1 procedure.updateTimestamp();
-1 queue.add(delayTask);
-1 } else {
-1 if (procedure.setTimeoutFailure(executor.getEnvironment())) {
-1 long rootProcId = executor.getRootProcedureId(procedure);
-1 RootProcedureStack<Env> rollbackStack = executor.getRollbackStack(rootProcId);
-1 rollbackStack.abort();
-1 executor.getStore().update(procedure);
-1 executor.getScheduler().addFront(procedure);
-1 }
-1 }
-1 }
-1 }
-1
-1 public void sendStopSignal() {}
-1
-1 private static class ProcedureDelayContainer<Env> implements Delayed {
-1
-1 private final Procedure<Env> procedure;
-1
-1 public ProcedureDelayContainer(Procedure<Env> procedure) {
-1 this.procedure = procedure;
-1 }
-1
-1 public Procedure<Env> getProcedure() {
-1 return procedure;
-1 }
-1
-1 @Override
-1 public long getDelay(TimeUnit unit) {
-1 long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis();
-1 return unit.convert(delay, TimeUnit.MILLISECONDS);
-1 }
-1
-1 @Override
-1 public int compareTo(Delayed other) {
-1 return Long.compare(
-1 this.getDelay(TimeUnit.MILLISECONDS), other.getDelay(TimeUnit.MILLISECONDS));
-1 }
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure;
+
+import java.util.concurrent.DelayQueue;
+import java.util.concurrent.Delayed;
+import java.util.concurrent.TimeUnit;
+
+public class TimeoutExecutorThread<Env> extends StoppableThread {
+
+ private static final int DELAY_QUEUE_TIMEOUT = 20;
+ private final ProcedureExecutor<Env> executor;
+ private final DelayQueue<ProcedureDelayContainer<Env>> queue = new DelayQueue<>();
+
+ public TimeoutExecutorThread(
+ ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, String name) {
+ super(threadGroup, name);
+ setDaemon(true);
+ this.executor = envProcedureExecutor;
+ }
+
+ public void add(Procedure<Env> procedure) {
+ queue.add(new ProcedureDelayContainer<>(procedure));
+ }
+
+ public boolean remove(Procedure<Env> procedure) {
+ return queue.remove(new ProcedureDelayContainer<>(procedure));
+ }
+
+ private ProcedureDelayContainer<Env> takeQuietly() {
+ try {
+ return queue.poll(DELAY_QUEUE_TIMEOUT, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ currentThread().interrupt();
+ return null;
+ }
+ }
+
+ @Override
+ public void run() {
+ while (executor.isRunning()) {
+ ProcedureDelayContainer<Env> delayTask = takeQuietly();
+ if (delayTask == null) {
+ continue;
+ }
+ Procedure<Env> procedure = delayTask.getProcedure();
+ if (procedure instanceof InternalProcedure) {
+ InternalProcedure internal = (InternalProcedure) procedure;
+ internal.periodicExecute(executor.getEnvironment());
+ procedure.updateTimestamp();
+ queue.add(delayTask);
+ } else {
+ if (procedure.setTimeoutFailure(executor.getEnvironment())) {
+ long rootProcId = executor.getRootProcedureId(procedure);
+ RootProcedureStack<Env> rollbackStack = executor.getRollbackStack(rootProcId);
+ rollbackStack.abort();
+ executor.getStore().update(procedure);
+ executor.getScheduler().addFront(procedure);
+ }
+ }
+ }
+ }
+
+ public void sendStopSignal() {}
+
+ private static class ProcedureDelayContainer<Env> implements Delayed {
+
+ private final Procedure<Env> procedure;
+
+ public ProcedureDelayContainer(Procedure<Env> procedure) {
+ this.procedure = procedure;
+ }
+
+ public Procedure<Env> getProcedure() {
+ return procedure;
+ }
+
+ @Override
+ public long getDelay(TimeUnit unit) {
+ long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis();
+ return unit.convert(delay, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public int compareTo(Delayed other) {
+ return Long.compare(
+ this.getDelay(TimeUnit.MILLISECONDS), other.getDelay(TimeUnit.MILLISECONDS));
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index f234032107b..960d0a7977f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -1,895 +1,894 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1
-1package org.apache.iotdb.confignode.procedure.env;
-1
-1import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-1import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-1import org.apache.iotdb.common.rpc.thrift.TSStatus;
-1import org.apache.iotdb.commons.cluster.NodeStatus;
-1import org.apache.iotdb.commons.cluster.NodeType;
-1import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
-1import org.apache.iotdb.commons.pipe.config.PipeConfig;
-1import org.apache.iotdb.commons.trigger.TriggerInformation;
-1import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
-1import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
-1import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
-1import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
-1import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
-1import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
-1import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
-1import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan;
-1import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
-1import org.apache.iotdb.confignode.exception.AddConsensusGroupException;
-1import org.apache.iotdb.confignode.exception.AddPeerException;
-1import org.apache.iotdb.confignode.manager.ConfigManager;
-1import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
-1import org.apache.iotdb.confignode.manager.load.LoadManager;
-1import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
-1import org.apache.iotdb.confignode.manager.node.NodeManager;
-1import org.apache.iotdb.confignode.manager.partition.PartitionManager;
-1import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
-1import org.apache.iotdb.confignode.persistence.partition.PartitionInfo;
-1import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo;
-1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-1import org.apache.iotdb.confignode.procedure.scheduler.LockQueue;
-1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-1import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
-1import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
-1import org.apache.iotdb.consensus.exception.ConsensusException;
-1import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
-1import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TCreatePipePluginInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushMultiTopicMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
-1import org.apache.iotdb.rpc.TSStatusCode;
-1
-1import org.apache.thrift.TException;
-1import org.apache.tsfile.utils.Binary;
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.io.IOException;
-1import java.nio.ByteBuffer;
-1import java.util.ArrayList;
-1import java.util.Arrays;
-1import java.util.HashMap;
-1import java.util.List;
-1import java.util.Map;
-1import java.util.concurrent.TimeUnit;
-1import java.util.concurrent.locks.ReentrantLock;
-1import java.util.stream.Collectors;
-1
-1public class ConfigNodeProcedureEnv {
-1
-1 private static final Logger LOG = LoggerFactory.getLogger(ConfigNodeProcedureEnv.class);
-1
-1 /** Add or remove node lock. */
-1 private final LockQueue nodeLock = new LockQueue();
-1
-1 private final ReentrantLock schedulerLock = new ReentrantLock(true);
-1
-1 private final ReentrantLock submitRegionMigrateLock = new ReentrantLock(true);
-1
-1 private final ConfigManager configManager;
-1
-1 private final ProcedureScheduler scheduler;
-1
-1 private final RegionMaintainHandler regionMaintainHandler;
-1
-1 private final RemoveDataNodeHandler removeDataNodeHandler;
-1
-1 private final ReentrantLock removeConfigNodeLock;
-1
-1 public ConfigNodeProcedureEnv(ConfigManager configManager, ProcedureScheduler scheduler) {
-1 this.configManager = configManager;
-1 this.scheduler = scheduler;
-1 this.regionMaintainHandler = new RegionMaintainHandler(configManager);
-1 this.removeDataNodeHandler = new RemoveDataNodeHandler(configManager);
-1 this.removeConfigNodeLock = new ReentrantLock();
-1 }
-1
-1 public ConfigManager getConfigManager() {
-1 return configManager;
-1 }
-1
-1 /**
-1 * Delete ConfigNode cache, includes {@link ClusterSchemaInfo} and {@link PartitionInfo}.
-1 *
-1 * @param name database name
-1 * @param isGeneratedByPipe whether the deletion is triggered by pipe request
-1 * @return tsStatus
-1 */
-1 public TSStatus deleteDatabaseConfig(final String name, final boolean isGeneratedByPipe) {
-1 return getClusterSchemaManager()
-1 .deleteDatabase(new DeleteDatabasePlan(name), isGeneratedByPipe);
-1 }
-1
-1 /**
-1 * Pre delete a database.
-1 *
-1 * @param preDeleteType execute/rollback
-1 * @param deleteSgName database name
-1 */
-1 public void preDeleteDatabase(
-1 final PreDeleteDatabasePlan.PreDeleteType preDeleteType, final String deleteSgName) {
-1 getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType);
-1 }
-1
-1 /**
-1 * @param databaseName database name
-1 * @return ALL SUCCESS OR NOT
-1 * @throws IOException IOE
-1 * @throws TException Thrift IOE
-1 */
-1 public boolean invalidateCache(final String databaseName) throws IOException, TException {
-1 final List<TDataNodeConfiguration> allDataNodes = getNodeManager().getRegisteredDataNodes();
-1 final TInvalidateCacheReq invalidateCacheReq = new TInvalidateCacheReq();
-1 invalidateCacheReq.setStorageGroup(true);
-1 invalidateCacheReq.setFullPath(databaseName);
-1 for (final TDataNodeConfiguration dataNodeConfiguration : allDataNodes) {
-1 final int dataNodeId = dataNodeConfiguration.getLocation().getDataNodeId();
-1
-1 // If the node is not alive, retry for up to 10 times
-1 NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
-1 int retryNum = 10;
-1 if (nodeStatus == NodeStatus.Unknown) {
-1 for (int i = 0; i < retryNum && nodeStatus == NodeStatus.Unknown; i++) {
-1 try {
-1 TimeUnit.MILLISECONDS.sleep(500);
-1 } catch (final InterruptedException e) {
-1 LOG.error("Sleep failed in ConfigNodeProcedureEnv: ", e);
-1 Thread.currentThread().interrupt();
-1 break;
-1 }
-1 nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
-1 }
-1 }
-1
-1 if (nodeStatus == NodeStatus.Unknown) {
-1 LOG.warn(
-1 "Invalidate cache failed, because DataNode {} is Unknown",
-1 dataNodeConfiguration.getLocation().getInternalEndPoint());
-1 return false;
-1 }
-1
-1 // Always invalidate PartitionCache first
-1 final TSStatus invalidatePartitionStatus =
-1 (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 dataNodeConfiguration.getLocation().getInternalEndPoint(),
-1 invalidateCacheReq,
-1 CnToDnSyncRequestType.INVALIDATE_PARTITION_CACHE);
-1
-1 final TSStatus invalidateSchemaStatus =
-1 (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 dataNodeConfiguration.getLocation().getInternalEndPoint(),
-1 invalidateCacheReq,
-1 CnToDnSyncRequestType.INVALIDATE_SCHEMA_CACHE);
-1
-1 if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) {
-1 LOG.error(
-1 "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine cache status is {}",
-1 invalidatePartitionStatus,
-1 invalidateSchemaStatus);
-1 return false;
-1 }
-1 }
-1 return true;
-1 }
-1
-1 public boolean verifySucceed(TSStatus... status) {
-1 return Arrays.stream(status)
-1 .allMatch(tsStatus -> tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode());
-1 }
-1
-1 /**
-1 * Let the remotely new ConfigNode build the ConsensusGroup.
-1 *
-1 * <p>Actually, the parameter of this method can be empty, adding new raft peer to exist group
-1 * should invoke createPeer(groupId, emptyList).
-1 *
-1 * @param tConfigNodeLocation New ConfigNode's location
-1 */
-1 public void addConsensusGroup(TConfigNodeLocation tConfigNodeLocation)
-1 throws AddConsensusGroupException {
-1 List<TConfigNodeLocation> configNodeLocations =
-1 new ArrayList<>(configManager.getNodeManager().getRegisteredConfigNodes());
-1 configNodeLocations.add(tConfigNodeLocation);
-1 TSStatus status =
-1 (TSStatus)
-1 SyncConfigNodeClientPool.getInstance()
-1 .sendSyncRequestToConfigNodeWithRetry(
-1 tConfigNodeLocation.getInternalEndPoint(),
-1 new TAddConsensusGroupReq(configNodeLocations),
-1 CnToCnNodeRequestType.ADD_CONSENSUS_GROUP);
-1 if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 throw new AddConsensusGroupException(tConfigNodeLocation);
-1 }
-1 }
-1
-1 /**
-1 * Leader will add the new ConfigNode Peer into ConfigRegion.
-1 *
-1 * @param configNodeLocation The new ConfigNode
-1 * @throws AddPeerException When addPeer doesn't success
-1 */
-1 public void addConfigNodePeer(TConfigNodeLocation configNodeLocation) throws AddPeerException {
-1 configManager.getConsensusManager().addConfigNodePeer(configNodeLocation);
-1 }
-1
-1 /**
-1 * Remove peer in Leader node.
-1 *
-1 * @param tConfigNodeLocation node is removed
-1 * @throws ProcedureException if failed status
-1 */
-1 public void removeConfigNodePeer(TConfigNodeLocation tConfigNodeLocation)
-1 throws ProcedureException {
-1 removeConfigNodeLock.lock();
-1 TSStatus tsStatus;
-1 try {
-1 // Execute removePeer
-1 if (getConsensusManager().removeConfigNodePeer(tConfigNodeLocation)) {
-1 tsStatus = getConsensusManager().write(new RemoveConfigNodePlan(tConfigNodeLocation));
-1 } else {
-1 tsStatus =
-1 new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode())
-1 .setMessage(
-1 "Remove ConfigNode failed because update ConsensusGroup peer information failed.");
-1 }
-1 } catch (ConsensusException e) {
-1 LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
-1 tsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
-1 tsStatus.setMessage(e.getMessage());
-1 }
-1 try {
-1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 throw new ProcedureException(tsStatus.getMessage());
-1 }
-1 } finally {
-1 removeConfigNodeLock.unlock();
-1 }
-1 }
-1
-1 /**
-1 * Remove Consensus Group in removed node.
-1 *
-1 * @param removedConfigNode config node location
-1 * @throws ProcedureException if failed status
-1 */
-1 public void deleteConfigNodePeer(TConfigNodeLocation removedConfigNode)
-1 throws ProcedureException {
-1 TSStatus tsStatus =
-1 (TSStatus)
-1 SyncConfigNodeClientPool.getInstance()
-1 .sendSyncRequestToConfigNodeWithRetry(
-1 removedConfigNode.getInternalEndPoint(),
-1 removedConfigNode,
-1 CnToCnNodeRequestType.DELETE_CONFIG_NODE_PEER);
-1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 throw new ProcedureException(tsStatus.getMessage());
-1 }
-1 }
-1
-1 /**
-1 * Stop ConfigNode and remove heartbeatCache.
-1 *
-1 * @param tConfigNodeLocation config node location
-1 * @throws ProcedureException if failed status
-1 */
-1 public void stopAndClearConfigNode(TConfigNodeLocation tConfigNodeLocation)
-1 throws ProcedureException {
-1 TSStatus tsStatus =
-1 (TSStatus)
-1 SyncConfigNodeClientPool.getInstance()
-1 .sendSyncRequestToConfigNodeWithRetry(
-1 tConfigNodeLocation.getInternalEndPoint(),
-1 tConfigNodeLocation,
-1 CnToCnNodeRequestType.STOP_AND_CLEAR_CONFIG_NODE);
-1
-1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 throw new ProcedureException(tsStatus.getMessage());
-1 }
-1
-1 getLoadManager().removeNodeCache(tConfigNodeLocation.getConfigNodeId());
-1 }
-1
-1 /**
-1 * Leader will record the new ConfigNode's information.
-1 *
-1 * @param configNodeLocation The new ConfigNode
-1 * @param versionInfo The new ConfigNode's versionInfo
-1 */
-1 public void applyConfigNode(
-1 TConfigNodeLocation configNodeLocation, TNodeVersionInfo versionInfo) {
-1 configManager.getNodeManager().applyConfigNode(configNodeLocation, versionInfo);
-1 }
-1
-1 /**
-1 * Leader will notify the new ConfigNode that registration success.
-1 *
-1 * @param configNodeLocation The new ConfigNode
-1 */
-1 public void notifyRegisterSuccess(TConfigNodeLocation configNodeLocation) {
-1 SyncConfigNodeClientPool.getInstance()
-1 .sendSyncRequestToConfigNodeWithRetry(
-1 configNodeLocation.getInternalEndPoint(),
-1 null,
-1 CnToCnNodeRequestType.NOTIFY_REGISTER_SUCCESS);
-1 }
-1
-1 /**
-1 * Create a new ConfigNodeHeartbeatCache
-1 *
-1 * @param nodeId the index of the new ConfigNode
-1 */
-1 public void createConfigNodeHeartbeatCache(int nodeId) {
-1 getLoadManager().getLoadCache().createNodeHeartbeatCache(NodeType.ConfigNode, nodeId);
-1 // TODO: invoke a force heartbeat to update new ConfigNode's status immediately
-1 }
-1
-1 /**
-1 * Do region creations and broadcast the {@link CreateRegionGroupsPlan}.
-1 *
-1 * @return Those RegionReplicas that failed to create
-1 */
-1 public Map<TConsensusGroupId, TRegionReplicaSet> doRegionCreation(
-1 final TConsensusGroupType consensusGroupType,
-1 final CreateRegionGroupsPlan createRegionGroupsPlan) {
-1
-1 // Prepare clientHandler
-1 DataNodeAsyncRequestContext<?, TSStatus> clientHandler;
-1 switch (consensusGroupType) {
-1 case SchemaRegion:
-1 clientHandler = getCreateSchemaRegionClientHandler(createRegionGroupsPlan);
-1 break;
-1 case DataRegion:
-1 default:
-1 clientHandler = getCreateDataRegionClientHandler(createRegionGroupsPlan);
-1 break;
-1 }
-1 if (clientHandler.getRequestIndices().isEmpty()) {
-1 return new HashMap<>();
-1 }
-1
-1 // Send CreateRegion requests to DataNodes
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1
-1 // Filter RegionGroups that weren't created successfully
-1 int requestId = 0;
-1 Map<Integer, TSStatus> responseMap = clientHandler.getResponseMap();
-1 Map<TConsensusGroupId, TRegionReplicaSet> failedRegions = new HashMap<>();
-1 for (List<TRegionReplicaSet> regionReplicaSets :
-1 createRegionGroupsPlan.getRegionGroupMap().values()) {
-1 for (final TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
-1 for (final TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
-1 if (responseMap.get(requestId).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 failedRegions
-1 .computeIfAbsent(
-1 regionReplicaSet.getRegionId(),
-1 empty -> new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()))
-1 .addToDataNodeLocations(dataNodeLocation);
-1 }
-1 requestId += 1;
-1 }
-1 }
-1 }
-1 return failedRegions;
-1 }
-1
-1 private DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus>
-1 getCreateSchemaRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
-1 DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_SCHEMA_REGION);
-1
-1 int requestId = 0;
-1 for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
-1 createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
-1 String storageGroup = sgRegionsEntry.getKey();
-1 List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
-1 for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
-1 for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
-1 clientHandler.putRequest(
-1 requestId, genCreateSchemaRegionReq(storageGroup, regionReplicaSet));
-1 clientHandler.putNodeLocation(requestId, dataNodeLocation);
-1 requestId += 1;
-1 }
-1 }
-1 }
-1
-1 return clientHandler;
-1 }
-1
-1 private DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus>
-1 getCreateDataRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
-1 DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_DATA_REGION);
-1
-1 int requestId = 0;
-1 for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
-1 createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
-1 String storageGroup = sgRegionsEntry.getKey();
-1 List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
-1 for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
-1 for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
-1 clientHandler.putRequest(
-1 requestId, genCreateDataRegionReq(storageGroup, regionReplicaSet));
-1 clientHandler.putNodeLocation(requestId, dataNodeLocation);
-1 requestId += 1;
-1 }
-1 }
-1 }
-1
-1 return clientHandler;
-1 }
-1
-1 private TCreateSchemaRegionReq genCreateSchemaRegionReq(
-1 String storageGroup, TRegionReplicaSet regionReplicaSet) {
-1 TCreateSchemaRegionReq req = new TCreateSchemaRegionReq();
-1 req.setStorageGroup(storageGroup);
-1 req.setRegionReplicaSet(regionReplicaSet);
-1 return req;
-1 }
-1
-1 private TCreateDataRegionReq genCreateDataRegionReq(
-1 String storageGroup, TRegionReplicaSet regionReplicaSet) {
-1 TCreateDataRegionReq req = new TCreateDataRegionReq();
-1 req.setStorageGroup(storageGroup);
-1 req.setRegionReplicaSet(regionReplicaSet);
-1 return req;
-1 }
-1
-1 public List<TSStatus> notifyRegionMigrationToAllDataNodes(
-1 TConsensusGroupId consensusGroupId, boolean isStart) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TNotifyRegionMigrationReq request =
-1 new TNotifyRegionMigrationReq(
-1 configManager
-1 .getConsensusManager()
-1 .getConsensusImpl()
-1 .getLogicalClock(ConfigNodeInfo.CONFIG_REGION_ID),
-1 System.nanoTime(),
-1 configManager.getProcedureManager().getRegionOperationConsensusIds());
-1 request.setRegionId(consensusGroupId);
-1 request.setIsStart(isStart);
-1
-1 final DataNodeAsyncRequestContext<TNotifyRegionMigrationReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.NOTIFY_REGION_MIGRATION, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public void persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
-1 // Persist the allocation result
-1 try {
-1 getConsensusManager().write(createRegionGroupsPlan);
-1 } catch (ConsensusException e) {
-1 LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
-1 }
-1 }
-1
-1 /**
-1 * Force activating RegionGroup by setting status to Running, therefore the ConfigNode-leader can
-1 * select leader for it and use it to allocate new Partitions
-1 *
-1 * @param activateRegionGroupMap Map<Database, Map<RegionGroupId, Map<DataNodeId, activate
-1 * heartbeat sample>>>
-1 */
-1 public void activateRegionGroup(
-1 Map<String, Map<TConsensusGroupId, Map<Integer, RegionHeartbeatSample>>>
-1 activateRegionGroupMap) {
-1 // Create RegionGroup heartbeat Caches
-1 activateRegionGroupMap.forEach(
-1 (database, regionGroupSampleMap) ->
-1 regionGroupSampleMap.forEach(
-1 (regionGroupId, regionSampleMap) ->
-1 getLoadManager()
-1 .getLoadCache()
-1 .createRegionGroupHeartbeatCache(
-1 database, regionGroupId, regionSampleMap.keySet())));
-1 // Force update first heartbeat samples
-1 getLoadManager()
-1 .forceUpdateRegionGroupCache(
-1 activateRegionGroupMap.values().stream()
-1 .flatMap(innerMap -> innerMap.entrySet().stream())
-1 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b)));
-1 // Wait for leader and priority redistribution
-1 getLoadManager()
-1 .waitForRegionGroupReady(
-1 activateRegionGroupMap.values().stream()
-1 .flatMap(innterMap -> innterMap.keySet().stream())
-1 .collect(Collectors.toList()));
-1 }
-1
-1 public List<TRegionReplicaSet> getAllReplicaSets(String storageGroup) {
-1 return getPartitionManager().getAllReplicaSets(storageGroup);
-1 }
-1
-1 public List<TSStatus> createTriggerOnDataNodes(
-1 TriggerInformation triggerInformation, Binary jarFile) throws IOException {
-1 NodeManager nodeManager = configManager.getNodeManager();
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 nodeManager.getRegisteredDataNodeLocations();
-1 final TCreateTriggerInstanceReq request =
-1 new TCreateTriggerInstanceReq(triggerInformation.serialize());
-1 if (jarFile != null) {
-1 request.setJarFile(ByteBuffer.wrap(jarFile.getValues()));
-1 }
-1
-1 DataNodeAsyncRequestContext<TCreateTriggerInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.CREATE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public List<TSStatus> dropTriggerOnDataNodes(String triggerName, boolean needToDeleteJarFile) {
-1 NodeManager nodeManager = configManager.getNodeManager();
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 nodeManager.getRegisteredDataNodeLocations();
-1 final TDropTriggerInstanceReq request =
-1 new TDropTriggerInstanceReq(triggerName, needToDeleteJarFile);
-1
-1 DataNodeAsyncRequestContext<TDropTriggerInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.DROP_TRIGGER_INSTANCE, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public List<TSStatus> activeTriggerOnDataNodes(String triggerName) {
-1 NodeManager nodeManager = configManager.getNodeManager();
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 nodeManager.getRegisteredDataNodeLocations();
-1 final TActiveTriggerInstanceReq request = new TActiveTriggerInstanceReq(triggerName);
-1
-1 DataNodeAsyncRequestContext<TActiveTriggerInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.ACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public List<TSStatus> inactiveTriggerOnDataNodes(String triggerName) {
-1 NodeManager nodeManager = configManager.getNodeManager();
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 nodeManager.getRegisteredDataNodeLocations();
-1 final TInactiveTriggerInstanceReq request = new TInactiveTriggerInstanceReq(triggerName);
-1
-1 DataNodeAsyncRequestContext<TInactiveTriggerInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.INACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public List<TSStatus> createPipePluginOnDataNodes(PipePluginMeta pipePluginMeta, byte[] jarFile)
-1 throws IOException {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TCreatePipePluginInstanceReq request =
-1 new TCreatePipePluginInstanceReq(pipePluginMeta.serialize(), ByteBuffer.wrap(jarFile));
-1
-1 final DataNodeAsyncRequestContext<TCreatePipePluginInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.CREATE_PIPE_PLUGIN, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public List<TSStatus> dropPipePluginOnDataNodes(
-1 String pipePluginName, boolean needToDeleteJarFile) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TDropPipePluginInstanceReq request =
-1 new TDropPipePluginInstanceReq(pipePluginName, needToDeleteJarFile);
-1
-1 DataNodeAsyncRequestContext<TDropPipePluginInstanceReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.DROP_PIPE_PLUGIN, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList();
-1 }
-1
-1 public Map<Integer, TPushPipeMetaResp> pushAllPipeMetaToDataNodes(
-1 List<ByteBuffer> pipeMetaBinaryList) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushPipeMetaReq request = new TPushPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
-1
-1 final DataNodeAsyncRequestContext<TPushPipeMetaReq, TPushPipeMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.PIPE_PUSH_ALL_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushPipeMetaResp> pushSinglePipeMetaToDataNodes(ByteBuffer pipeMetaBinary) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSinglePipeMetaReq request = new TPushSinglePipeMetaReq().setPipeMeta(pipeMetaBinary);
-1
-1 final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushPipeMetaResp> dropSinglePipeOnDataNodes(String pipeNameToDrop) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSinglePipeMetaReq request =
-1 new TPushSinglePipeMetaReq().setPipeNameToDrop(pipeNameToDrop);
-1
-1 final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushPipeMetaResp> pushMultiPipeMetaToDataNodes(
-1 List<ByteBuffer> pipeMetaBinaryList) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushMultiPipeMetaReq request =
-1 new TPushMultiPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
-1
-1 final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushPipeMetaResp> dropMultiPipeOnDataNodes(List<String> pipeNamesToDrop) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushMultiPipeMetaReq request =
-1 new TPushMultiPipeMetaReq().setPipeNamesToDrop(pipeNamesToDrop);
-1
-1 final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushTopicMetaResp> pushAllTopicMetaToDataNodes(
-1 List<ByteBuffer> topicMetaBinaryList) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushTopicMetaReq request = new TPushTopicMetaReq().setTopicMetas(topicMetaBinaryList);
-1
-1 final DataNodeAsyncRequestContext<TPushTopicMetaReq, TPushTopicMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.TOPIC_PUSH_ALL_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public List<TSStatus> pushSingleTopicOnDataNode(ByteBuffer topicMeta) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSingleTopicMetaReq request = new TPushSingleTopicMetaReq().setTopicMeta(topicMeta);
-1
-1 final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList().stream()
-1 .map(TPushTopicMetaResp::getStatus)
-1 .collect(Collectors.toList());
-1 }
-1
-1 public List<TSStatus> dropSingleTopicOnDataNode(String topicNameToDrop) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSingleTopicMetaReq request =
-1 new TPushSingleTopicMetaReq().setTopicNameToDrop(topicNameToDrop);
-1
-1 final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList().stream()
-1 .map(TPushTopicMetaResp::getStatus)
-1 .collect(Collectors.toList());
-1 }
-1
-1 public Map<Integer, TPushTopicMetaResp> pushMultiTopicMetaToDataNodes(
-1 List<ByteBuffer> topicMetaBinaryList) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushMultiTopicMetaReq request =
-1 new TPushMultiTopicMetaReq().setTopicMetas(topicMetaBinaryList);
-1
-1 final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushTopicMetaResp> dropMultiTopicOnDataNodes(List<String> topicNamesToDrop) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushMultiTopicMetaReq request =
-1 new TPushMultiTopicMetaReq().setTopicNamesToDrop(topicNamesToDrop);
-1
-1 final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public Map<Integer, TPushConsumerGroupMetaResp> pushAllConsumerGroupMetaToDataNodes(
-1 List<ByteBuffer> consumerGroupMetaBinaryList) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushConsumerGroupMetaReq request =
-1 new TPushConsumerGroupMetaReq().setConsumerGroupMetas(consumerGroupMetaBinaryList);
-1
-1 final DataNodeAsyncRequestContext<TPushConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
-1 clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_ALL_META, request, dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance()
-1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
-1 clientHandler,
-1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 public List<TSStatus> pushSingleConsumerGroupOnDataNode(ByteBuffer consumerGroupMeta) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSingleConsumerGroupMetaReq request =
-1 new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(consumerGroupMeta);
-1
-1 final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
-1 clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
-1 request,
-1 dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList().stream()
-1 .map(TPushConsumerGroupMetaResp::getStatus)
-1 .collect(Collectors.toList());
-1 }
-1
-1 public List<TSStatus> dropSingleConsumerGroupOnDataNode(String consumerGroupNameToDrop) {
-1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
-1 configManager.getNodeManager().getRegisteredDataNodeLocations();
-1 final TPushSingleConsumerGroupMetaReq request =
-1 new TPushSingleConsumerGroupMetaReq().setConsumerGroupNameToDrop(consumerGroupNameToDrop);
-1
-1 final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
-1 clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
-1 request,
-1 dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseList().stream()
-1 .map(TPushConsumerGroupMetaResp::getStatus)
-1 .collect(Collectors.toList());
-1 }
-1
-1 public LockQueue getNodeLock() {
-1 return nodeLock;
-1 }
-1
-1 public ProcedureScheduler getScheduler() {
-1 return scheduler;
-1 }
-1
-1 public ReentrantLock getSchedulerLock() {
-1 return schedulerLock;
-1 }
-1
-1 public ReentrantLock getSubmitRegionMigrateLock() {
-1 return submitRegionMigrateLock;
-1 }
-1
-1 public RegionMaintainHandler getRegionMaintainHandler() {
-1 return regionMaintainHandler;
-1 }
-1
-1 public RemoveDataNodeHandler getRemoveDataNodeHandler() {
-1 return removeDataNodeHandler;
-1 }
-1
-1 private ConsensusManager getConsensusManager() {
-1 return configManager.getConsensusManager();
-1 }
-1
-1 private NodeManager getNodeManager() {
-1 return configManager.getNodeManager();
-1 }
-1
-1 private ClusterSchemaManager getClusterSchemaManager() {
-1 return configManager.getClusterSchemaManager();
-1 }
-1
-1 private PartitionManager getPartitionManager() {
-1 return configManager.getPartitionManager();
-1 }
-1
-1 private LoadManager getLoadManager() {
-1 return configManager.getLoadManager();
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.env;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.NodeType;
+import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
+import org.apache.iotdb.commons.pipe.config.PipeConfig;
+import org.apache.iotdb.commons.trigger.TriggerInformation;
+import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
+import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
+import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
+import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
+import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan;
+import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
+import org.apache.iotdb.confignode.exception.AddConsensusGroupException;
+import org.apache.iotdb.confignode.exception.AddPeerException;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
+import org.apache.iotdb.confignode.persistence.partition.PartitionInfo;
+import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo;
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.confignode.procedure.scheduler.LockQueue;
+import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
+import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
+import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
+import org.apache.iotdb.mpp.rpc.thrift.TCreatePipePluginInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
+import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq;
+import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
+import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
+import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushMultiTopicMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp;
+import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
+import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.apache.tsfile.utils.Binary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+
+public class ConfigNodeProcedureEnv {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ConfigNodeProcedureEnv.class);
+
+ /** Add or remove node lock. */
+ private final LockQueue nodeLock = new LockQueue();
+
+ private final ReentrantLock schedulerLock = new ReentrantLock(true);
+
+ private final ReentrantLock submitRegionMigrateLock = new ReentrantLock(true);
+
+ private final ConfigManager configManager;
+
+ private final ProcedureScheduler scheduler;
+
+ private final RegionMaintainHandler regionMaintainHandler;
+
+ private final RemoveDataNodeHandler removeDataNodeHandler;
+
+ private final ReentrantLock removeConfigNodeLock;
+
+ public ConfigNodeProcedureEnv(ConfigManager configManager, ProcedureScheduler scheduler) {
+ this.configManager = configManager;
+ this.scheduler = scheduler;
+ this.regionMaintainHandler = new RegionMaintainHandler(configManager);
+ this.removeDataNodeHandler = new RemoveDataNodeHandler(configManager);
+ this.removeConfigNodeLock = new ReentrantLock();
+ }
+
+ public ConfigManager getConfigManager() {
+ return configManager;
+ }
+
+ /**
+ * Delete ConfigNode cache, includes {@link ClusterSchemaInfo} and {@link PartitionInfo}.
+ *
+ * @param name database name
+ * @param isGeneratedByPipe whether the deletion is triggered by pipe request
+ * @return tsStatus
+ */
+ public TSStatus deleteDatabaseConfig(final String name, final boolean isGeneratedByPipe) {
+ return getClusterSchemaManager()
+ .deleteDatabase(new DeleteDatabasePlan(name), isGeneratedByPipe);
+ }
+
+ /**
+ * Pre delete a database.
+ *
+ * @param preDeleteType execute/rollback
+ * @param deleteSgName database name
+ */
+ public void preDeleteDatabase(
+ final PreDeleteDatabasePlan.PreDeleteType preDeleteType, final String deleteSgName) {
+ getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType);
+ }
+
+ /**
+ * @param databaseName database name
+ * @return ALL SUCCESS OR NOT
+ * @throws IOException IOE
+ * @throws TException Thrift IOE
+ */
+ public boolean invalidateCache(final String databaseName) throws IOException, TException {
+ final List<TDataNodeConfiguration> allDataNodes = getNodeManager().getRegisteredDataNodes();
+ final TInvalidateCacheReq invalidateCacheReq = new TInvalidateCacheReq();
+ invalidateCacheReq.setStorageGroup(true);
+ invalidateCacheReq.setFullPath(databaseName);
+ for (final TDataNodeConfiguration dataNodeConfiguration : allDataNodes) {
+ final int dataNodeId = dataNodeConfiguration.getLocation().getDataNodeId();
+
+ // If the node is not alive, retry for up to 10 times
+ NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
+ int retryNum = 10;
+ if (nodeStatus == NodeStatus.Unknown) {
+ for (int i = 0; i < retryNum && nodeStatus == NodeStatus.Unknown; i++) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(500);
+ } catch (final InterruptedException e) {
+ LOG.error("Sleep failed in ConfigNodeProcedureEnv: ", e);
+ Thread.currentThread().interrupt();
+ break;
+ }
+ nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
+ }
+ }
+
+ if (nodeStatus == NodeStatus.Unknown) {
+ LOG.warn(
+ "Invalidate cache failed, because DataNode {} is Unknown",
+ dataNodeConfiguration.getLocation().getInternalEndPoint());
+ return false;
+ }
+
+ // Always invalidate PartitionCache first
+ final TSStatus invalidatePartitionStatus =
+ (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ dataNodeConfiguration.getLocation().getInternalEndPoint(),
+ invalidateCacheReq,
+ CnToDnSyncRequestType.INVALIDATE_PARTITION_CACHE);
+
+ final TSStatus invalidateSchemaStatus =
+ (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ dataNodeConfiguration.getLocation().getInternalEndPoint(),
+ invalidateCacheReq,
+ CnToDnSyncRequestType.INVALIDATE_SCHEMA_CACHE);
+
+ if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) {
+ LOG.error(
+ "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine cache status is {}",
+ invalidatePartitionStatus,
+ invalidateSchemaStatus);
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean verifySucceed(TSStatus... status) {
+ return Arrays.stream(status)
+ .allMatch(tsStatus -> tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ /**
+ * Let the remotely new ConfigNode build the ConsensusGroup.
+ *
+ * <p>Actually, the parameter of this method can be empty, adding new raft peer to exist group
+ * should invoke createPeer(groupId, emptyList).
+ *
+ * @param tConfigNodeLocation New ConfigNode's location
+ */
+ public void addConsensusGroup(TConfigNodeLocation tConfigNodeLocation)
+ throws AddConsensusGroupException {
+ List<TConfigNodeLocation> configNodeLocations =
+ new ArrayList<>(configManager.getNodeManager().getRegisteredConfigNodes());
+ configNodeLocations.add(tConfigNodeLocation);
+ TSStatus status =
+ (TSStatus)
+ SyncConfigNodeClientPool.getInstance()
+ .sendSyncRequestToConfigNodeWithRetry(
+ tConfigNodeLocation.getInternalEndPoint(),
+ new TAddConsensusGroupReq(configNodeLocations),
+ CnToCnNodeRequestType.ADD_CONSENSUS_GROUP);
+ if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new AddConsensusGroupException(tConfigNodeLocation);
+ }
+ }
+
+ /**
+ * Leader will add the new ConfigNode Peer into ConfigRegion.
+ *
+ * @param configNodeLocation The new ConfigNode
+ * @throws AddPeerException When addPeer doesn't success
+ */
+ public void addConfigNodePeer(TConfigNodeLocation configNodeLocation) throws AddPeerException {
+ configManager.getConsensusManager().addConfigNodePeer(configNodeLocation);
+ }
+
+ /**
+ * Remove peer in Leader node.
+ *
+ * @param tConfigNodeLocation node is removed
+ * @throws ProcedureException if failed status
+ */
+ public void removeConfigNodePeer(TConfigNodeLocation tConfigNodeLocation)
+ throws ProcedureException {
+ removeConfigNodeLock.lock();
+ TSStatus tsStatus;
+ try {
+ // Execute removePeer
+ if (getConsensusManager().removeConfigNodePeer(tConfigNodeLocation)) {
+ tsStatus = getConsensusManager().write(new RemoveConfigNodePlan(tConfigNodeLocation));
+ } else {
+ tsStatus =
+ new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode())
+ .setMessage(
+ "Remove ConfigNode failed because update ConsensusGroup peer information failed.");
+ }
+ } catch (ConsensusException e) {
+ LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
+ tsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ tsStatus.setMessage(e.getMessage());
+ }
+ try {
+ if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new ProcedureException(tsStatus.getMessage());
+ }
+ } finally {
+ removeConfigNodeLock.unlock();
+ }
+ }
+
+ /**
+ * Remove Consensus Group in removed node.
+ *
+ * @param removedConfigNode config node location
+ * @throws ProcedureException if failed status
+ */
+ public void deleteConfigNodePeer(TConfigNodeLocation removedConfigNode)
+ throws ProcedureException {
+ TSStatus tsStatus =
+ (TSStatus)
+ SyncConfigNodeClientPool.getInstance()
+ .sendSyncRequestToConfigNodeWithRetry(
+ removedConfigNode.getInternalEndPoint(),
+ removedConfigNode,
+ CnToCnNodeRequestType.DELETE_CONFIG_NODE_PEER);
+ if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new ProcedureException(tsStatus.getMessage());
+ }
+ }
+
+ /**
+ * Stop ConfigNode and remove heartbeatCache.
+ *
+ * @param tConfigNodeLocation config node location
+ * @throws ProcedureException if failed status
+ */
+ public void stopAndClearConfigNode(TConfigNodeLocation tConfigNodeLocation)
+ throws ProcedureException {
+ TSStatus tsStatus =
+ (TSStatus)
+ SyncConfigNodeClientPool.getInstance()
+ .sendSyncRequestToConfigNodeWithRetry(
+ tConfigNodeLocation.getInternalEndPoint(),
+ tConfigNodeLocation,
+ CnToCnNodeRequestType.STOP_AND_CLEAR_CONFIG_NODE);
+
+ if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ throw new ProcedureException(tsStatus.getMessage());
+ }
+
+ getLoadManager().removeNodeCache(tConfigNodeLocation.getConfigNodeId());
+ }
+
+ /**
+ * Leader will record the new ConfigNode's information.
+ *
+ * @param configNodeLocation The new ConfigNode
+ * @param versionInfo The new ConfigNode's versionInfo
+ */
+ public void applyConfigNode(
+ TConfigNodeLocation configNodeLocation, TNodeVersionInfo versionInfo) {
+ configManager.getNodeManager().applyConfigNode(configNodeLocation, versionInfo);
+ }
+
+ /**
+ * Leader will notify the new ConfigNode that registration success.
+ *
+ * @param configNodeLocation The new ConfigNode
+ */
+ public void notifyRegisterSuccess(TConfigNodeLocation configNodeLocation) {
+ SyncConfigNodeClientPool.getInstance()
+ .sendSyncRequestToConfigNodeWithRetry(
+ configNodeLocation.getInternalEndPoint(),
+ null,
+ CnToCnNodeRequestType.NOTIFY_REGISTER_SUCCESS);
+ }
+
+ /**
+ * Create a new ConfigNodeHeartbeatCache
+ *
+ * @param nodeId the index of the new ConfigNode
+ */
+ public void createConfigNodeHeartbeatCache(int nodeId) {
+ getLoadManager().getLoadCache().createNodeHeartbeatCache(NodeType.ConfigNode, nodeId);
+ // TODO: invoke a force heartbeat to update new ConfigNode's status immediately
+ }
+
+ /**
+ * Do region creations and broadcast the {@link CreateRegionGroupsPlan}.
+ *
+ * @return Those RegionReplicas that failed to create
+ */
+ public Map<TConsensusGroupId, TRegionReplicaSet> doRegionCreation(
+ final TConsensusGroupType consensusGroupType,
+ final CreateRegionGroupsPlan createRegionGroupsPlan) {
+
+ // Prepare clientHandler
+ DataNodeAsyncRequestContext<?, TSStatus> clientHandler;
+ switch (consensusGroupType) {
+ case SchemaRegion:
+ clientHandler = getCreateSchemaRegionClientHandler(createRegionGroupsPlan);
+ break;
+ case DataRegion:
+ default:
+ clientHandler = getCreateDataRegionClientHandler(createRegionGroupsPlan);
+ break;
+ }
+ if (clientHandler.getRequestIndices().isEmpty()) {
+ return new HashMap<>();
+ }
+
+ // Send CreateRegion requests to DataNodes
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+
+ // Filter RegionGroups that weren't created successfully
+ int requestId = 0;
+ Map<Integer, TSStatus> responseMap = clientHandler.getResponseMap();
+ Map<TConsensusGroupId, TRegionReplicaSet> failedRegions = new HashMap<>();
+ for (List<TRegionReplicaSet> regionReplicaSets :
+ createRegionGroupsPlan.getRegionGroupMap().values()) {
+ for (final TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+ for (final TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+ if (responseMap.get(requestId).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ failedRegions
+ .computeIfAbsent(
+ regionReplicaSet.getRegionId(),
+ empty -> new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()))
+ .addToDataNodeLocations(dataNodeLocation);
+ }
+ requestId += 1;
+ }
+ }
+ }
+ return failedRegions;
+ }
+
+ private DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus>
+ getCreateSchemaRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
+ DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_SCHEMA_REGION);
+
+ int requestId = 0;
+ for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
+ createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
+ String storageGroup = sgRegionsEntry.getKey();
+ List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
+ for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+ for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+ clientHandler.putRequest(
+ requestId, genCreateSchemaRegionReq(storageGroup, regionReplicaSet));
+ clientHandler.putNodeLocation(requestId, dataNodeLocation);
+ requestId += 1;
+ }
+ }
+ }
+
+ return clientHandler;
+ }
+
+ private DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus>
+ getCreateDataRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
+ DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_DATA_REGION);
+
+ int requestId = 0;
+ for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
+ createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
+ String storageGroup = sgRegionsEntry.getKey();
+ List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
+ for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+ for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+ clientHandler.putRequest(
+ requestId, genCreateDataRegionReq(storageGroup, regionReplicaSet));
+ clientHandler.putNodeLocation(requestId, dataNodeLocation);
+ requestId += 1;
+ }
+ }
+ }
+
+ return clientHandler;
+ }
+
+ private TCreateSchemaRegionReq genCreateSchemaRegionReq(
+ String storageGroup, TRegionReplicaSet regionReplicaSet) {
+ TCreateSchemaRegionReq req = new TCreateSchemaRegionReq();
+ req.setStorageGroup(storageGroup);
+ req.setRegionReplicaSet(regionReplicaSet);
+ return req;
+ }
+
+ private TCreateDataRegionReq genCreateDataRegionReq(
+ String storageGroup, TRegionReplicaSet regionReplicaSet) {
+ TCreateDataRegionReq req = new TCreateDataRegionReq();
+ req.setStorageGroup(storageGroup);
+ req.setRegionReplicaSet(regionReplicaSet);
+ return req;
+ }
+
+ public List<TSStatus> notifyRegionMigrationToAllDataNodes(
+ TConsensusGroupId consensusGroupId, boolean isStart) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TNotifyRegionMigrationReq request =
+ new TNotifyRegionMigrationReq(
+ configManager
+ .getConsensusManager()
+ .getConsensusImpl()
+ .getLogicalClock(ConfigNodeInfo.CONFIG_REGION_ID),
+ System.nanoTime(),
+ configManager.getProcedureManager().getRegionOperationConsensusIds());
+ request.setRegionId(consensusGroupId);
+ request.setIsStart(isStart);
+
+ final DataNodeAsyncRequestContext<TNotifyRegionMigrationReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.NOTIFY_REGION_MIGRATION, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public void persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
+ // Persist the allocation result
+ try {
+ getConsensusManager().write(createRegionGroupsPlan);
+ } catch (ConsensusException e) {
+ LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
+ }
+ }
+
+ /**
+ * Force activating RegionGroup by setting status to Running, therefore the ConfigNode-leader can
+ * select leader for it and use it to allocate new Partitions
+ *
+ * @param activateRegionGroupMap Map<Database, Map<RegionGroupId, Map<DataNodeId, activate
+ * heartbeat sample>>>
+ */
+ public void activateRegionGroup(
+ Map<String, Map<TConsensusGroupId, Map<Integer, RegionHeartbeatSample>>>
+ activateRegionGroupMap) {
+ // Create RegionGroup heartbeat Caches
+ activateRegionGroupMap.forEach(
+ (database, regionGroupSampleMap) ->
+ regionGroupSampleMap.forEach(
+ (regionGroupId, regionSampleMap) ->
+ getLoadManager()
+ .getLoadCache()
+ .createRegionGroupHeartbeatCache(
+ database, regionGroupId, regionSampleMap.keySet())));
+ // Force update first heartbeat samples
+ getLoadManager()
+ .forceUpdateRegionGroupCache(
+ activateRegionGroupMap.values().stream()
+ .flatMap(innerMap -> innerMap.entrySet().stream())
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b)));
+ // Wait for leader and priority redistribution
+ getLoadManager()
+ .waitForRegionGroupReady(
+ activateRegionGroupMap.values().stream()
+ .flatMap(innterMap -> innterMap.keySet().stream())
+ .collect(Collectors.toList()));
+ }
+
+ public List<TRegionReplicaSet> getAllReplicaSets(String storageGroup) {
+ return getPartitionManager().getAllReplicaSets(storageGroup);
+ }
+
+ public List<TSStatus> createTriggerOnDataNodes(
+ TriggerInformation triggerInformation, Binary jarFile) throws IOException {
+ NodeManager nodeManager = configManager.getNodeManager();
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ nodeManager.getRegisteredDataNodeLocations();
+ final TCreateTriggerInstanceReq request =
+ new TCreateTriggerInstanceReq(triggerInformation.serialize());
+ if (jarFile != null) {
+ request.setJarFile(ByteBuffer.wrap(jarFile.getValues()));
+ }
+
+ DataNodeAsyncRequestContext<TCreateTriggerInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.CREATE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public List<TSStatus> dropTriggerOnDataNodes(String triggerName, boolean needToDeleteJarFile) {
+ NodeManager nodeManager = configManager.getNodeManager();
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ nodeManager.getRegisteredDataNodeLocations();
+ final TDropTriggerInstanceReq request =
+ new TDropTriggerInstanceReq(triggerName, needToDeleteJarFile);
+
+ DataNodeAsyncRequestContext<TDropTriggerInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.DROP_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public List<TSStatus> activeTriggerOnDataNodes(String triggerName) {
+ NodeManager nodeManager = configManager.getNodeManager();
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ nodeManager.getRegisteredDataNodeLocations();
+ final TActiveTriggerInstanceReq request = new TActiveTriggerInstanceReq(triggerName);
+
+ DataNodeAsyncRequestContext<TActiveTriggerInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.ACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public List<TSStatus> inactiveTriggerOnDataNodes(String triggerName) {
+ NodeManager nodeManager = configManager.getNodeManager();
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ nodeManager.getRegisteredDataNodeLocations();
+ final TInactiveTriggerInstanceReq request = new TInactiveTriggerInstanceReq(triggerName);
+
+ DataNodeAsyncRequestContext<TInactiveTriggerInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.INACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public List<TSStatus> createPipePluginOnDataNodes(PipePluginMeta pipePluginMeta, byte[] jarFile)
+ throws IOException {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TCreatePipePluginInstanceReq request =
+ new TCreatePipePluginInstanceReq(pipePluginMeta.serialize(), ByteBuffer.wrap(jarFile));
+
+ final DataNodeAsyncRequestContext<TCreatePipePluginInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.CREATE_PIPE_PLUGIN, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public List<TSStatus> dropPipePluginOnDataNodes(
+ String pipePluginName, boolean needToDeleteJarFile) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TDropPipePluginInstanceReq request =
+ new TDropPipePluginInstanceReq(pipePluginName, needToDeleteJarFile);
+
+ DataNodeAsyncRequestContext<TDropPipePluginInstanceReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.DROP_PIPE_PLUGIN, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList();
+ }
+
+ public Map<Integer, TPushPipeMetaResp> pushAllPipeMetaToDataNodes(
+ List<ByteBuffer> pipeMetaBinaryList) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushPipeMetaReq request = new TPushPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
+
+ final DataNodeAsyncRequestContext<TPushPipeMetaReq, TPushPipeMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PIPE_PUSH_ALL_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushPipeMetaResp> pushSinglePipeMetaToDataNodes(ByteBuffer pipeMetaBinary) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSinglePipeMetaReq request = new TPushSinglePipeMetaReq().setPipeMeta(pipeMetaBinary);
+
+ final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushPipeMetaResp> dropSinglePipeOnDataNodes(String pipeNameToDrop) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSinglePipeMetaReq request =
+ new TPushSinglePipeMetaReq().setPipeNameToDrop(pipeNameToDrop);
+
+ final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushPipeMetaResp> pushMultiPipeMetaToDataNodes(
+ List<ByteBuffer> pipeMetaBinaryList) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushMultiPipeMetaReq request =
+ new TPushMultiPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
+
+ final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushPipeMetaResp> dropMultiPipeOnDataNodes(List<String> pipeNamesToDrop) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushMultiPipeMetaReq request =
+ new TPushMultiPipeMetaReq().setPipeNamesToDrop(pipeNamesToDrop);
+
+ final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushTopicMetaResp> pushAllTopicMetaToDataNodes(
+ List<ByteBuffer> topicMetaBinaryList) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushTopicMetaReq request = new TPushTopicMetaReq().setTopicMetas(topicMetaBinaryList);
+
+ final DataNodeAsyncRequestContext<TPushTopicMetaReq, TPushTopicMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.TOPIC_PUSH_ALL_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public List<TSStatus> pushSingleTopicOnDataNode(ByteBuffer topicMeta) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSingleTopicMetaReq request = new TPushSingleTopicMetaReq().setTopicMeta(topicMeta);
+
+ final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList().stream()
+ .map(TPushTopicMetaResp::getStatus)
+ .collect(Collectors.toList());
+ }
+
+ public List<TSStatus> dropSingleTopicOnDataNode(String topicNameToDrop) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSingleTopicMetaReq request =
+ new TPushSingleTopicMetaReq().setTopicNameToDrop(topicNameToDrop);
+
+ final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList().stream()
+ .map(TPushTopicMetaResp::getStatus)
+ .collect(Collectors.toList());
+ }
+
+ public Map<Integer, TPushTopicMetaResp> pushMultiTopicMetaToDataNodes(
+ List<ByteBuffer> topicMetaBinaryList) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushMultiTopicMetaReq request =
+ new TPushMultiTopicMetaReq().setTopicMetas(topicMetaBinaryList);
+
+ final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushTopicMetaResp> dropMultiTopicOnDataNodes(List<String> topicNamesToDrop) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushMultiTopicMetaReq request =
+ new TPushMultiTopicMetaReq().setTopicNamesToDrop(topicNamesToDrop);
+
+ final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public Map<Integer, TPushConsumerGroupMetaResp> pushAllConsumerGroupMetaToDataNodes(
+ List<ByteBuffer> consumerGroupMetaBinaryList) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushConsumerGroupMetaReq request =
+ new TPushConsumerGroupMetaReq().setConsumerGroupMetas(consumerGroupMetaBinaryList);
+
+ final DataNodeAsyncRequestContext<TPushConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+ clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_ALL_META, request, dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance()
+ .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+ clientHandler,
+ PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+ return clientHandler.getResponseMap();
+ }
+
+ public List<TSStatus> pushSingleConsumerGroupOnDataNode(ByteBuffer consumerGroupMeta) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSingleConsumerGroupMetaReq request =
+ new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(consumerGroupMeta);
+
+ final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+ clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
+ request,
+ dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList().stream()
+ .map(TPushConsumerGroupMetaResp::getStatus)
+ .collect(Collectors.toList());
+ }
+
+ public List<TSStatus> dropSingleConsumerGroupOnDataNode(String consumerGroupNameToDrop) {
+ final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+ configManager.getNodeManager().getRegisteredDataNodeLocations();
+ final TPushSingleConsumerGroupMetaReq request =
+ new TPushSingleConsumerGroupMetaReq().setConsumerGroupNameToDrop(consumerGroupNameToDrop);
+
+ final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+ clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
+ request,
+ dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseList().stream()
+ .map(TPushConsumerGroupMetaResp::getStatus)
+ .collect(Collectors.toList());
+ }
+
+ public LockQueue getNodeLock() {
+ return nodeLock;
+ }
+
+ public ProcedureScheduler getScheduler() {
+ return scheduler;
+ }
+
+ public ReentrantLock getSchedulerLock() {
+ return schedulerLock;
+ }
+
+ public ReentrantLock getSubmitRegionMigrateLock() {
+ return submitRegionMigrateLock;
+ }
+
+ public RegionMaintainHandler getRegionMaintainHandler() {
+ return regionMaintainHandler;
+ }
+
+ public RemoveDataNodeHandler getRemoveDataNodeHandler() {
+ return removeDataNodeHandler;
+ }
+
+ private ConsensusManager getConsensusManager() {
+ return configManager.getConsensusManager();
+ }
+
+ private NodeManager getNodeManager() {
+ return configManager.getNodeManager();
+ }
+
+ private ClusterSchemaManager getClusterSchemaManager() {
+ return configManager.getClusterSchemaManager();
+ }
+
+ private PartitionManager getPartitionManager() {
+ return configManager.getPartitionManager();
+ }
+
+ private LoadManager getLoadManager() {
+ return configManager.getLoadManager();
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
index ebea1555bc1..928f3cfcd37 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
@@ -1,588 +1,587 @@
-1/*
-1 * Licensed to the Apache Software Foundation (ASF) under one
-1 * or more contributor license agreements. See the NOTICE file
-1 * distributed with this work for additional information
-1 * regarding copyright ownership. The ASF licenses this file
-1 * to you under the Apache License, Version 2.0 (the
-1 * "License"); you may not use this file except in compliance
-1 * with the License. You may obtain a copy of the License at
-1 *
-1 * http://www.apache.org/licenses/LICENSE-2.0
-1 *
-1 * Unless required by applicable law or agreed to in writing,
-1 * software distributed under the License is distributed on an
-1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-1 * KIND, either express or implied. See the License for the
-1 * specific language governing permissions and limitations
-1 * under the License.
-1 */
-1package org.apache.iotdb.confignode.procedure.env;
-1
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-1import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-1import org.apache.iotdb.common.rpc.thrift.TRegionMaintainTaskStatus;
-1import org.apache.iotdb.common.rpc.thrift.TRegionMigrateFailedType;
-1import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-1import org.apache.iotdb.common.rpc.thrift.TSStatus;
-1import org.apache.iotdb.commons.client.ClientPoolFactory;
-1import org.apache.iotdb.commons.client.IClientManager;
-1import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
-1import org.apache.iotdb.commons.cluster.NodeStatus;
-1import org.apache.iotdb.commons.cluster.RegionStatus;
-1import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
-1import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
-1import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
-1import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
-1import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
-1import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
-1import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
-1import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
-1import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan;
-1import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan;
-1import org.apache.iotdb.confignode.manager.ConfigManager;
-1import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
-1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-1import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq;
-1import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp;
-1import org.apache.iotdb.mpp.rpc.thrift.TRegionMigrateResult;
-1import org.apache.iotdb.mpp.rpc.thrift.TResetPeerListReq;
-1import org.apache.iotdb.rpc.TSStatusCode;
-1
-1import org.slf4j.Logger;
-1import org.slf4j.LoggerFactory;
-1
-1import java.util.ArrayList;
-1import java.util.Collections;
-1import java.util.HashMap;
-1import java.util.List;
-1import java.util.Map;
-1import java.util.Optional;
-1import java.util.concurrent.TimeUnit;
-1import java.util.stream.Collectors;
-1
-1import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REGION_MIGRATE_PROCESS;
-1import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS;
-1import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS_V2;
-1import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS;
-1
-1public class RegionMaintainHandler {
-1
-1 private static final Logger LOGGER = LoggerFactory.getLogger(RegionMaintainHandler.class);
-1
-1 private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
-1
-1 private final ConfigManager configManager;
-1
-1 private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> dataNodeClientManager;
-1
-1 public RegionMaintainHandler(ConfigManager configManager) {
-1 this.configManager = configManager;
-1 dataNodeClientManager =
-1 new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
-1 .createClientManager(
-1 new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
-1 }
-1
-1 public static String getIdWithRpcEndpoint(TDataNodeLocation location) {
-1 return String.format(
-1 "[dataNodeId: %s, clientRpcEndPoint: %s]",
-1 location.getDataNodeId(), location.getClientRpcEndPoint());
-1 }
-1
-1 public static String simplifiedLocation(TDataNodeLocation dataNodeLocation) {
-1 return dataNodeLocation.getDataNodeId() + "@" + dataNodeLocation.getInternalEndPoint().getIp();
-1 }
-1
-1 /**
-1 * Find dest data node.
-1 *
-1 * @param regionId region id
-1 * @return dest data node location
-1 */
-1 public TDataNodeLocation findDestDataNode(TConsensusGroupId regionId) {
-1 TSStatus status;
-1 List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
-1 if (regionReplicaNodes.isEmpty()) {
-1 LOGGER.warn("Cannot find region replica nodes, region: {}", regionId);
-1 status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
-1 status.setMessage("Cannot find region replica nodes, region: " + regionId);
-1 return null;
-1 }
-1
-1 Optional<TDataNodeLocation> newNode = pickNewReplicaNodeForRegion(regionReplicaNodes);
-1 if (!newNode.isPresent()) {
-1 LOGGER.warn("No enough Data node to migrate region: {}", regionId);
-1 return null;
-1 }
-1 return newNode.get();
-1 }
-1
-1 /**
-1 * Create a new RegionReplica and build the ConsensusGroup on the destined DataNode
-1 *
-1 * <p>createNewRegionPeer should be invoked on a DataNode that doesn't contain any peer of the
-1 * specific ConsensusGroup, in order to avoid there exists one DataNode who has more than one
-1 * RegionReplica.
-1 *
-1 * @param regionId The given ConsensusGroup
-1 * @param destDataNode The destined DataNode where the new peer will be created
-1 * @return status
-1 */
-1 public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocation destDataNode) {
-1 TSStatus status;
-1 List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
-1 if (regionReplicaNodes.isEmpty()) {
-1 LOGGER.warn(
-1 "{}, Cannot find region replica nodes in createPeer, regionId: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId);
-1 status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
-1 status.setMessage("Not find region replica nodes in createPeer, regionId: " + regionId);
-1 return status;
-1 }
-1
-1 List<TDataNodeLocation> currentPeerNodes;
-1 if (TConsensusGroupType.DataRegion.equals(regionId.getType())
-1 && (IOT_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())
-1 || IOT_CONSENSUS_V2.equals(CONF.getDataRegionConsensusProtocolClass()))) {
-1 // parameter of createPeer for MultiLeader should be all peers
-1 currentPeerNodes = new ArrayList<>(regionReplicaNodes);
-1 currentPeerNodes.add(destDataNode);
-1 } else {
-1 // parameter of createPeer for Ratis can be empty
-1 currentPeerNodes = Collections.emptyList();
-1 }
-1
-1 String database = configManager.getPartitionManager().getRegionDatabase(regionId);
-1 TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, database);
-1
-1 status =
-1 (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 destDataNode.getInternalEndPoint(),
-1 req,
-1 CnToDnSyncRequestType.CREATE_NEW_REGION_PEER);
-1
-1 if (isSucceed(status)) {
-1 LOGGER.info(
-1 "{}, Send action createNewRegionPeer finished, regionId: {}, newPeerDataNodeId: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 getIdWithRpcEndpoint(destDataNode));
-1 } else {
-1 LOGGER.error(
-1 "{}, Send action createNewRegionPeer error, regionId: {}, newPeerDataNodeId: {}, result: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 getIdWithRpcEndpoint(destDataNode),
-1 status);
-1 }
-1
-1 return status;
-1 }
-1
-1 /**
-1 * Order the specific ConsensusGroup to add peer for the new RegionReplica.
-1 *
-1 * <p>The add peer interface could be invoked at any DataNode who contains one of the
-1 * RegionReplica of the specified ConsensusGroup except the new one
-1 *
-1 * @param destDataNode The DataNodeLocation where the new RegionReplica is created
-1 * @param regionId region id
-1 * @return TSStatus
-1 */
-1 public TSStatus submitAddRegionPeerTask(
-1 long procedureId,
-1 TDataNodeLocation destDataNode,
-1 TConsensusGroupId regionId,
-1 TDataNodeLocation coordinator) {
-1 TSStatus status;
-1
-1 // Send addRegionPeer request to the selected DataNode,
-1 // destDataNode is where the new RegionReplica is created
-1 TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId);
-1 status =
-1 (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 coordinator.getInternalEndPoint(),
-1 maintainPeerReq,
-1 CnToDnSyncRequestType.ADD_REGION_PEER);
-1 LOGGER.info(
-1 "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 getIdWithRpcEndpoint(coordinator),
-1 getIdWithRpcEndpoint(destDataNode),
-1 status);
-1 return status;
-1 }
-1
-1 /**
-1 * Order the specific ConsensusGroup to remove peer for the old RegionReplica.
-1 *
-1 * <p>The remove peer interface could be invoked at any DataNode who contains one of the
-1 * RegionReplica of the specified ConsensusGroup except the origin one
-1 *
-1 * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
-1 * @param regionId region id
-1 * @return TSStatus
-1 */
-1 public TSStatus submitRemoveRegionPeerTask(
-1 long procedureId,
-1 TDataNodeLocation originalDataNode,
-1 TConsensusGroupId regionId,
-1 TDataNodeLocation coordinator) {
-1 TSStatus status;
-1
-1 // Send removeRegionPeer request to the rpcClientDataNode
-1 TMaintainPeerReq maintainPeerReq =
-1 new TMaintainPeerReq(regionId, originalDataNode, procedureId);
-1 status =
-1 (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 coordinator.getInternalEndPoint(),
-1 maintainPeerReq,
-1 CnToDnSyncRequestType.REMOVE_REGION_PEER);
-1 LOGGER.info(
-1 "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 getIdWithRpcEndpoint(coordinator));
-1 return status;
-1 }
-1
-1 /**
-1 * Delete a Region peer in the given ConsensusGroup and all of its data on the specified DataNode
-1 *
-1 * <p>If the originalDataNode is down, we should delete local data and do other cleanup works
-1 * manually.
-1 *
-1 * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
-1 * @param regionId region id
-1 * @return TSStatus
-1 */
-1 public TSStatus submitDeleteOldRegionPeerTask(
-1 long procedureId, TDataNodeLocation originalDataNode, TConsensusGroupId regionId) {
-1
-1 TSStatus status;
-1 TMaintainPeerReq maintainPeerReq =
-1 new TMaintainPeerReq(regionId, originalDataNode, procedureId);
-1
-1 status =
-1 configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId())
-1 == NodeStatus.Unknown
-1 ? (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithGivenRetry(
-1 originalDataNode.getInternalEndPoint(),
-1 maintainPeerReq,
-1 CnToDnSyncRequestType.DELETE_OLD_REGION_PEER,
-1 1)
-1 : (TSStatus)
-1 SyncDataNodeClientPool.getInstance()
-1 .sendSyncRequestToDataNodeWithRetry(
-1 originalDataNode.getInternalEndPoint(),
-1 maintainPeerReq,
-1 CnToDnSyncRequestType.DELETE_OLD_REGION_PEER);
-1 LOGGER.info(
-1 "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 originalDataNode.getInternalEndPoint());
-1 return status;
-1 }
-1
-1 public Map<Integer, TSStatus> resetPeerList(
-1 TConsensusGroupId regionId,
-1 List<TDataNodeLocation> correctDataNodeLocations,
-1 Map<Integer, TDataNodeLocation> dataNodeLocationMap) {
-1 DataNodeAsyncRequestContext<TResetPeerListReq, TSStatus> clientHandler =
-1 new DataNodeAsyncRequestContext<>(
-1 CnToDnAsyncRequestType.RESET_PEER_LIST,
-1 new TResetPeerListReq(regionId, correctDataNodeLocations),
-1 dataNodeLocationMap);
-1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-1 return clientHandler.getResponseMap();
-1 }
-1
-1 // TODO: will use 'procedure yield' to refactor later
-1 public TRegionMigrateResult waitTaskFinish(long taskId, TDataNodeLocation dataNodeLocation) {
-1 final long MAX_DISCONNECTION_TOLERATE_MS = 600_000;
-1 final long INITIAL_DISCONNECTION_TOLERATE_MS = 60_000;
-1 long startTime = System.nanoTime();
-1 long lastReportTime = System.nanoTime();
-1 while (true) {
-1 try (SyncDataNodeInternalServiceClient dataNodeClient =
-1 dataNodeClientManager.borrowClient(dataNodeLocation.getInternalEndPoint())) {
-1 TRegionMigrateResult report = dataNodeClient.getRegionMaintainResult(taskId);
-1 lastReportTime = System.nanoTime();
-1 if (report.getTaskStatus() != TRegionMaintainTaskStatus.PROCESSING) {
-1 return report;
-1 }
-1 } catch (Exception ignore) {
-1
-1 }
-1 long waitTime =
-1 Math.min(
-1 INITIAL_DISCONNECTION_TOLERATE_MS
-1 + TimeUnit.NANOSECONDS.toMillis(lastReportTime - startTime) / 60,
-1 MAX_DISCONNECTION_TOLERATE_MS);
-1 long disconnectionTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime);
-1 if (disconnectionTime > waitTime) {
-1 LOGGER.warn(
-1 "{} task {} cannot get task report from DataNode {}, last report time is {} ago",
-1 REGION_MIGRATE_PROCESS,
-1 taskId,
-1 dataNodeLocation,
-1 CommonDateTimeUtils.convertMillisecondToDurationStr(
-1 TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime)));
-1 TRegionMigrateResult report = new TRegionMigrateResult();
-1 report.setTaskStatus(TRegionMaintainTaskStatus.FAIL);
-1 report.setFailedNodeAndReason(new HashMap<>());
-1 report.getFailedNodeAndReason().put(dataNodeLocation, TRegionMigrateFailedType.Disconnect);
-1 return report;
-1 }
-1 try {
-1 TimeUnit.SECONDS.sleep(1);
-1 } catch (InterruptedException ignore) {
-1 Thread.currentThread().interrupt();
-1 return new TRegionMigrateResult(TRegionMaintainTaskStatus.PROCESSING);
-1 }
-1 }
-1 }
-1
-1 public void addRegionLocation(TConsensusGroupId regionId, TDataNodeLocation newLocation) {
-1 AddRegionLocationPlan req = new AddRegionLocationPlan(regionId, newLocation);
-1 TSStatus status = configManager.getPartitionManager().addRegionLocation(req);
-1 LOGGER.info(
-1 "AddRegionLocation finished, add region {} to {}, result is {}",
-1 regionId,
-1 getIdWithRpcEndpoint(newLocation),
-1 status);
-1 configManager
-1 .getLoadManager()
-1 .getLoadCache()
-1 .createRegionCache(regionId, newLocation.getDataNodeId());
-1 }
-1
-1 public void forceUpdateRegionCache(
-1 TConsensusGroupId regionId, TDataNodeLocation newLocation, RegionStatus regionStatus) {
-1 configManager
-1 .getLoadManager()
-1 .forceUpdateRegionCache(regionId, newLocation.getDataNodeId(), regionStatus);
-1 }
-1
-1 public void removeRegionLocation(
-1 TConsensusGroupId regionId, TDataNodeLocation deprecatedLocation) {
-1 RemoveRegionLocationPlan req = new RemoveRegionLocationPlan(regionId, deprecatedLocation);
-1 TSStatus status = configManager.getPartitionManager().removeRegionLocation(req);
-1 LOGGER.info(
-1 "RemoveRegionLocation remove region {} from DataNode {}, result is {}",
-1 regionId,
-1 getIdWithRpcEndpoint(deprecatedLocation),
-1 status);
-1 configManager.getLoadManager().removeRegionCache(regionId, deprecatedLocation.getDataNodeId());
-1 configManager.getLoadManager().getRouteBalancer().balanceRegionLeaderAndPriority();
-1 }
-1
-1 /**
-1 * Find all DataNodes which contains the given regionId
-1 *
-1 * @param regionId region id
-1 * @return DataNode locations
-1 */
-1 public List<TDataNodeLocation> findRegionLocations(TConsensusGroupId regionId) {
-1 Optional<TRegionReplicaSet> regionReplicaSet = getRegionReplicaSet(regionId);
-1 if (regionReplicaSet.isPresent()) {
-1 return regionReplicaSet.get().getDataNodeLocations();
-1 }
-1 return Collections.emptyList();
-1 }
-1
-1 public Optional<TRegionReplicaSet> getRegionReplicaSet(TConsensusGroupId regionId) {
-1 return configManager.getPartitionManager().getAllReplicaSets().stream()
-1 .filter(rg -> rg.regionId.equals(regionId))
-1 .findAny();
-1 }
-1
-1 public String getRegionReplicaSetString(TConsensusGroupId regionId) {
-1 Optional<TRegionReplicaSet> regionReplicaSet = getRegionReplicaSet(regionId);
-1 if (!regionReplicaSet.isPresent()) {
-1 return "UNKNOWN!";
-1 }
-1 StringBuilder result = new StringBuilder(regionReplicaSet.get().getRegionId() + ": {");
-1 for (TDataNodeLocation dataNodeLocation : regionReplicaSet.get().getDataNodeLocations()) {
-1 result.append(simplifiedLocation(dataNodeLocation)).append(", ");
-1 }
-1 result.append("}");
-1 return result.toString();
-1 }
-1
-1 private Optional<TDataNodeLocation> pickNewReplicaNodeForRegion(
-1 List<TDataNodeLocation> regionReplicaNodes) {
-1 List<TDataNodeConfiguration> dataNodeConfigurations =
-1 configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running);
-1 // Randomly selected to ensure a basic load balancing
-1 Collections.shuffle(dataNodeConfigurations);
-1 return dataNodeConfigurations.stream()
-1 .map(TDataNodeConfiguration::getLocation)
-1 .filter(e -> !regionReplicaNodes.contains(e))
-1 .findAny();
-1 }
-1
-1 public boolean isSucceed(TSStatus status) {
-1 return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
-1 }
-1
-1 public boolean isFailed(TSStatus status) {
-1 return !isSucceed(status);
-1 }
-1
-1 /**
-1 * Change the leader of given Region.
-1 *
-1 * <p>For IOT_CONSENSUS, using `changeLeaderForIoTConsensus` method to change the regionLeaderMap
-1 * maintained in ConfigNode.
-1 *
-1 * <p>For RATIS_CONSENSUS, invoking `changeRegionLeader` DataNode RPC method to change the leader.
-1 *
-1 * @param regionId The region to be migrated
-1 * @param originalDataNode The DataNode where the region locates
-1 */
-1 public void transferRegionLeader(
-1 TConsensusGroupId regionId, TDataNodeLocation originalDataNode, TDataNodeLocation coodinator)
-1 throws ProcedureException, InterruptedException {
-1 // find new leader
-1 Optional<TDataNodeLocation> newLeaderNode = Optional.empty();
-1 List<TDataNodeLocation> excludeDataNode = new ArrayList<>();
-1 excludeDataNode.add(originalDataNode);
-1 excludeDataNode.add(coodinator);
-1 newLeaderNode = filterDataNodeWithOtherRegionReplica(regionId, excludeDataNode);
-1 if (!newLeaderNode.isPresent()) {
-1 // If we have no choice, we use it
-1 newLeaderNode = Optional.of(coodinator);
-1 }
-1 // ratis needs DataNode to do election by itself
-1 long timestamp = System.nanoTime();
-1 if (TConsensusGroupType.SchemaRegion.equals(regionId.getType())
-1 || TConsensusGroupType.DataRegion.equals(regionId.getType())
-1 && RATIS_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())) {
-1 final int MAX_RETRY_TIME = 10;
-1 int retryTime = 0;
-1 long sleepTime =
-1 (CONF.getSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs()
-1 + CONF.getSchemaRegionRatisRpcLeaderElectionTimeoutMinMs())
-1 / 2;
-1 Integer leaderId = configManager.getLoadManager().getRegionLeaderMap().get(regionId);
-1
-1 if (leaderId != -1) {
-1 // The migrated node is not leader, so we don't need to transfer temporarily
-1 if (originalDataNode.getDataNodeId() != leaderId) {
-1 return;
-1 }
-1 }
-1 while (true) {
-1 TRegionLeaderChangeResp resp =
-1 SyncDataNodeClientPool.getInstance()
-1 .changeRegionLeader(
-1 regionId, originalDataNode.getInternalEndPoint(), newLeaderNode.get());
-1 if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-1 timestamp = resp.getConsensusLogicalTimestamp();
-1 break;
-1 }
-1 if (retryTime++ > MAX_RETRY_TIME) {
-1 LOGGER.warn("[RemoveRegion] Ratis transfer leader fail, but procedure will continue.");
-1 return;
-1 }
-1 LOGGER.warn(
-1 "Call changeRegionLeader fail for the {} time, will sleep {} ms", retryTime, sleepTime);
-1 Thread.sleep(sleepTime);
-1 }
-1 }
-1
-1 configManager
-1 .getLoadManager()
-1 .forceUpdateConsensusGroupCache(
-1 Collections.singletonMap(
-1 regionId,
-1 new ConsensusGroupHeartbeatSample(timestamp, newLeaderNode.get().getDataNodeId())));
-1 configManager.getLoadManager().getRouteBalancer().balanceRegionLeaderAndPriority();
-1
-1 LOGGER.info(
-1 "{}, Change region leader finished, regionId: {}, newLeaderNode: {}",
-1 REGION_MIGRATE_PROCESS,
-1 regionId,
-1 newLeaderNode);
-1 }
-1
-1 /**
-1 * Filter a DataNode who contains other RegionReplica excepts the given one.
-1 *
-1 * <p>Choose the RUNNING status datanode firstly, if no RUNNING status datanode match the
-1 * condition, then we choose the REMOVING status datanode.
-1 *
-1 * <p>`addRegionPeer`, `removeRegionPeer` and `changeRegionLeader` invoke this method.
-1 *
-1 * @param regionId The specific RegionId
-1 * @param filterLocation The DataNodeLocation that should be filtered
-1 * @return A DataNodeLocation that contains other RegionReplica and different from the
-1 * filterLocation
-1 */
-1 public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
-1 TConsensusGroupId regionId, TDataNodeLocation filterLocation) {
-1 List<TDataNodeLocation> filterLocations = Collections.singletonList(filterLocation);
-1 return filterDataNodeWithOtherRegionReplica(regionId, filterLocations);
-1 }
-1
-1 public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
-1 TConsensusGroupId regionId, List<TDataNodeLocation> filterLocations) {
-1 return filterDataNodeWithOtherRegionReplica(
-1 regionId, filterLocations, NodeStatus.Running, NodeStatus.ReadOnly);
-1 }
-1
-1 public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
-1 TConsensusGroupId regionId, TDataNodeLocation filterLocation, NodeStatus... allowingStatus) {
-1 List<TDataNodeLocation> excludeLocations = Collections.singletonList(filterLocation);
-1 return filterDataNodeWithOtherRegionReplica(regionId, excludeLocations, allowingStatus);
-1 }
-1
-1 public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
-1 TConsensusGroupId regionId,
-1 List<TDataNodeLocation> excludeLocations,
-1 NodeStatus... allowingStatus) {
-1 List<TDataNodeLocation> regionLocations = findRegionLocations(regionId);
-1 if (regionLocations.isEmpty()) {
-1 LOGGER.warn("Cannot find DataNodes contain the given region: {}", regionId);
-1 return Optional.empty();
-1 }
-1
-1 // Choosing the RUNNING DataNodes to execute firstly
-1 // If all DataNodes are not RUNNING, then choose the REMOVING DataNodes secondly
-1 List<TDataNodeLocation> aliveDataNodes =
-1 configManager.getNodeManager().filterDataNodeThroughStatus(allowingStatus).stream()
-1 .map(TDataNodeConfiguration::getLocation)
-1 .collect(Collectors.toList());
-1 final int leaderId = configManager.getLoadManager().getRegionLeaderMap().get(regionId);
-1 Collections.shuffle(aliveDataNodes);
-1 Optional<TDataNodeLocation> bestChoice = Optional.empty();
-1 for (TDataNodeLocation aliveDataNode : aliveDataNodes) {
-1 if (regionLocations.contains(aliveDataNode) && !excludeLocations.contains(aliveDataNode)) {
-1 if (leaderId == aliveDataNode.getDataNodeId()) {
-1 bestChoice = Optional.of(aliveDataNode);
-1 break;
-1 } else if (!bestChoice.isPresent()) {
-1 bestChoice = Optional.of(aliveDataNode);
-1 }
-1 }
-1 }
-1 return bestChoice;
-1 }
-1}
-1
\ No newline at end of file
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.procedure.env;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionMaintainTaskStatus;
+import org.apache.iotdb.common.rpc.thrift.TRegionMigrateFailedType;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.ClientPoolFactory;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.RegionStatus;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
+import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
+import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
+import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan;
+import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
+import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq;
+import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq;
+import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp;
+import org.apache.iotdb.mpp.rpc.thrift.TRegionMigrateResult;
+import org.apache.iotdb.mpp.rpc.thrift.TResetPeerListReq;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REGION_MIGRATE_PROCESS;
+import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS;
+import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS_V2;
+import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS;
+
+public class RegionMaintainHandler {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(RegionMaintainHandler.class);
+
+ private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
+
+ private final ConfigManager configManager;
+
+ private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> dataNodeClientManager;
+
+ public RegionMaintainHandler(ConfigManager configManager) {
+ this.configManager = configManager;
+ dataNodeClientManager =
+ new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
+ .createClientManager(
+ new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+ }
+
+ public static String getIdWithRpcEndpoint(TDataNodeLocation location) {
+ return String.format(
+ "[dataNodeId: %s, clientRpcEndPoint: %s]",
+ location.getDataNodeId(), location.getClientRpcEndPoint());
+ }
+
+ public static String simplifiedLocation(TDataNodeLocation dataNodeLocation) {
+ return dataNodeLocation.getDataNodeId() + "@" + dataNodeLocation.getInternalEndPoint().getIp();
+ }
+
+ /**
+ * Find dest data node.
+ *
+ * @param regionId region id
+ * @return dest data node location
+ */
+ public TDataNodeLocation findDestDataNode(TConsensusGroupId regionId) {
+ TSStatus status;
+ List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
+ if (regionReplicaNodes.isEmpty()) {
+ LOGGER.warn("Cannot find region replica nodes, region: {}", regionId);
+ status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ status.setMessage("Cannot find region replica nodes, region: " + regionId);
+ return null;
+ }
+
+ Optional<TDataNodeLocation> newNode = pickNewReplicaNodeForRegion(regionReplicaNodes);
+ if (!newNode.isPresent()) {
+ LOGGER.warn("No enough Data node to migrate region: {}", regionId);
+ return null;
+ }
+ return newNode.get();
+ }
+
+ /**
+ * Create a new RegionReplica and build the ConsensusGroup on the destined DataNode
+ *
+ * <p>createNewRegionPeer should be invoked on a DataNode that doesn't contain any peer of the
+ * specific ConsensusGroup, in order to avoid there exists one DataNode who has more than one
+ * RegionReplica.
+ *
+ * @param regionId The given ConsensusGroup
+ * @param destDataNode The destined DataNode where the new peer will be created
+ * @return status
+ */
+ public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocation destDataNode) {
+ TSStatus status;
+ List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
+ if (regionReplicaNodes.isEmpty()) {
+ LOGGER.warn(
+ "{}, Cannot find region replica nodes in createPeer, regionId: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId);
+ status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+ status.setMessage("Not find region replica nodes in createPeer, regionId: " + regionId);
+ return status;
+ }
+
+ List<TDataNodeLocation> currentPeerNodes;
+ if (TConsensusGroupType.DataRegion.equals(regionId.getType())
+ && (IOT_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())
+ || IOT_CONSENSUS_V2.equals(CONF.getDataRegionConsensusProtocolClass()))) {
+ // parameter of createPeer for MultiLeader should be all peers
+ currentPeerNodes = new ArrayList<>(regionReplicaNodes);
+ currentPeerNodes.add(destDataNode);
+ } else {
+ // parameter of createPeer for Ratis can be empty
+ currentPeerNodes = Collections.emptyList();
+ }
+
+ String database = configManager.getPartitionManager().getRegionDatabase(regionId);
+ TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, database);
+
+ status =
+ (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ destDataNode.getInternalEndPoint(),
+ req,
+ CnToDnSyncRequestType.CREATE_NEW_REGION_PEER);
+
+ if (isSucceed(status)) {
+ LOGGER.info(
+ "{}, Send action createNewRegionPeer finished, regionId: {}, newPeerDataNodeId: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId,
+ getIdWithRpcEndpoint(destDataNode));
+ } else {
+ LOGGER.error(
+ "{}, Send action createNewRegionPeer error, regionId: {}, newPeerDataNodeId: {}, result: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId,
+ getIdWithRpcEndpoint(destDataNode),
+ status);
+ }
+
+ return status;
+ }
+
+ /**
+ * Order the specific ConsensusGroup to add peer for the new RegionReplica.
+ *
+ * <p>The add peer interface could be invoked at any DataNode who contains one of the
+ * RegionReplica of the specified ConsensusGroup except the new one
+ *
+ * @param destDataNode The DataNodeLocation where the new RegionReplica is created
+ * @param regionId region id
+ * @return TSStatus
+ */
+ public TSStatus submitAddRegionPeerTask(
+ long procedureId,
+ TDataNodeLocation destDataNode,
+ TConsensusGroupId regionId,
+ TDataNodeLocation coordinator) {
+ TSStatus status;
+
+ // Send addRegionPeer request to the selected DataNode,
+ // destDataNode is where the new RegionReplica is created
+ TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId);
+ status =
+ (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ coordinator.getInternalEndPoint(),
+ maintainPeerReq,
+ CnToDnSyncRequestType.ADD_REGION_PEER);
+ LOGGER.info(
+ "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId,
+ getIdWithRpcEndpoint(coordinator),
+ getIdWithRpcEndpoint(destDataNode),
+ status);
+ return status;
+ }
+
+ /**
+ * Order the specific ConsensusGroup to remove peer for the old RegionReplica.
+ *
+ * <p>The remove peer interface could be invoked at any DataNode who contains one of the
+ * RegionReplica of the specified ConsensusGroup except the origin one
+ *
+ * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
+ * @param regionId region id
+ * @return TSStatus
+ */
+ public TSStatus submitRemoveRegionPeerTask(
+ long procedureId,
+ TDataNodeLocation originalDataNode,
+ TConsensusGroupId regionId,
+ TDataNodeLocation coordinator) {
+ TSStatus status;
+
+ // Send removeRegionPeer request to the rpcClientDataNode
+ TMaintainPeerReq maintainPeerReq =
+ new TMaintainPeerReq(regionId, originalDataNode, procedureId);
+ status =
+ (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ coordinator.getInternalEndPoint(),
+ maintainPeerReq,
+ CnToDnSyncRequestType.REMOVE_REGION_PEER);
+ LOGGER.info(
+ "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId,
+ getIdWithRpcEndpoint(coordinator));
+ return status;
+ }
+
+ /**
+ * Delete a Region peer in the given ConsensusGroup and all of its data on the specified DataNode
+ *
+ * <p>If the originalDataNode is down, we should delete local data and do other cleanup works
+ * manually.
+ *
+ * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
+ * @param regionId region id
+ * @return TSStatus
+ */
+ public TSStatus submitDeleteOldRegionPeerTask(
+ long procedureId, TDataNodeLocation originalDataNode, TConsensusGroupId regionId) {
+
+ TSStatus status;
+ TMaintainPeerReq maintainPeerReq =
+ new TMaintainPeerReq(regionId, originalDataNode, procedureId);
+
+ status =
+ configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId())
+ == NodeStatus.Unknown
+ ? (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithGivenRetry(
+ originalDataNode.getInternalEndPoint(),
+ maintainPeerReq,
+ CnToDnSyncRequestType.DELETE_OLD_REGION_PEER,
+ 1)
+ : (TSStatus)
+ SyncDataNodeClientPool.getInstance()
+ .sendSyncRequestToDataNodeWithRetry(
+ originalDataNode.getInternalEndPoint(),
+ maintainPeerReq,
+ CnToDnSyncRequestType.DELETE_OLD_REGION_PEER);
+ LOGGER.info(
+ "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}",
+ REGION_MIGRATE_PROCESS,
+ regionId,
+ originalDataNode.getInternalEndPoint());
+ return status;
+ }
+
+ public Map<Integer, TSStatus> resetPeerList(
+ TConsensusGroupId regionId,
+ List<TDataNodeLocation> correctDataNodeLocations,
+ Map<Integer, TDataNodeLocation> dataNodeLocationMap) {
+ DataNodeAsyncRequestContext<TResetPeerListReq, TSStatus> clientHandler =
+ new DataNodeAsyncRequestContext<>(
+ CnToDnAsyncRequestType.RESET_PEER_LIST,
+ new TResetPeerListReq(regionId, correctDataNodeLocations),
+ dataNodeLocationMap);
+ CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+ return clientHandler.getResponseMap();
+ }
+
+ // TODO: will use 'procedure yield' to refactor later
+ public TRegionMigrateResult waitTaskFinish(long taskId, TDataNodeLocation dataNodeLocation) {
+ final long MAX_DISCONNECTION_TOLERATE_MS = 600_000;
+ final long INITIAL_DISCONNECTION_TOLERATE_MS = 60_000;
+ long startTime = System.nanoTime();
+ long lastReportTime = System.nanoTime();
+ while (true) {
+ try (SyncDataNodeInternalServiceClient dataNodeClient =
+ dataNodeClientManager.borrowClient(dataNodeLocation.getInternalEndPoint())) {
+ TRegionMigrateResult report = dataNodeClient.getRegionMaintainResult(taskId);
+ lastReportTime = System.nanoTime();
+ if (report.getTaskStatus() != TRegionMaintainTaskStatus.PROCESSING) {
+ return report;
+ }
+ } catch (Exception ignore) {
+
+ }
+ long waitTime =
+ Math.min(
+ INITIAL_DISCONNECTION_TOLERATE_MS
+ + TimeUnit.NANOSECONDS.toMillis(lastReportTime - startTime) / 60,
+ MAX_DISCONNECTION_TOLERATE_MS);
+ long disconnectionTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime);
+ if (disconnectionTime > waitTime) {
+ LOGGER.warn(
+ "{} task {} cannot get task report from DataNode {}, last report time is {} ago",
+ REGION_MIGRATE_PROCESS,
+ taskId,
+ dataNodeLocation,
+ CommonDateTimeUtils.convertMillisecondToDurationStr(
+ TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime)));
+ TRegionMigrateResult report = new TRegionMigrateResult();
+ report.setTaskStatus(TRegionMaintainTaskStatus.FAIL);
+ report.setFailedNodeAndReason(new HashMap<>());
+ report.getFailedNodeAndReason().put(dataNodeLocation, TRegionMigrateFailedType.Disconnect);
+ return report;
+ }
+ try {
+ TimeUnit.SECONDS.sleep(1);
+ } catch (InterruptedException ignore) {
+ Thread.currentThread().interrupt();
+ return new TRegionMigrateResult(TRegionMaintainTaskStatus.PROCESSING);
+ }
+ }
+ }
+
+ public void addRegionLocation(TConsensusGroupId regionId, TDataNodeLocation newLocation) {
+ AddRegionLocationPlan req = new AddRegionLocationPlan(regionId, newLocation);
+ TSStatus status = configManager.getPartitionManager().addRegionLocation(req);
+ LOGGER.info(
+ "AddRegionLocation finished, add region {} to {}, result is {}",
+ regionId,
... 4877 lines suppressed ...
(iotdb) 01/02: reformat procedure
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8160a18727ac0d06846725e59ea8666277a608c3
Author: Yongzao <53...@qq.com>
AuthorDate: Sun Sep 28 11:59:43 2025 +0800
reformat procedure
---
.../iotdb/confignode/manager/ProcedureManager.java | 2000 ++++++++++----------
.../procedure/CompletedProcedureContainer.java | 73 +-
.../procedure/CompletedProcedureRecycler.java | 169 +-
.../confignode/procedure/InternalProcedure.java | 99 +-
.../procedure/PartitionTableAutoCleaner.java | 225 +--
.../iotdb/confignode/procedure/Procedure.java | 1647 ++++++++--------
.../confignode/procedure/ProcedureExecutor.java | 1881 +++++++++---------
.../confignode/procedure/ProcedureMetrics.java | 369 ++--
.../confignode/procedure/RootProcedureStack.java | 355 ++--
.../confignode/procedure/StoppableThread.java | 111 +-
.../procedure/TimeoutExecutorThread.java | 215 +--
.../procedure/env/ConfigNodeProcedureEnv.java | 1789 ++++++++---------
.../procedure/env/RegionMaintainHandler.java | 1175 ++++++------
.../procedure/env/RemoveDataNodeHandler.java | 1363 ++++++-------
.../procedure/exception/ProcedureException.java | 89 +-
.../scheduler/AbstractProcedureScheduler.java | 441 ++---
.../confignode/procedure/scheduler/LockQueue.java | 119 +-
.../procedure/scheduler/ProcedureScheduler.java | 223 +--
.../scheduler/SimpleProcedureScheduler.java | 159 +-
.../procedure/store/ConfigProcedureStore.java | 363 ++--
.../procedure/store/IProcedureFactory.java | 61 +-
.../procedure/store/IProcedureStore.java | 117 +-
.../procedure/store/ProcedureFactory.java | 1119 +++++------
.../confignode/procedure/store/ProcedureType.java | 377 ++--
.../confignode/procedure/store/ProcedureWAL.java | 153 +-
25 files changed, 7358 insertions(+), 7334 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index 3dcb01de273..96ee4e1c680 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -1,1003 +1,1003 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.manager;
-
-import org.apache.iotdb.common.rpc.thrift.Model;
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonConfig;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.conf.IoTDBConstant;
-import org.apache.iotdb.commons.exception.IoTDBException;
-import org.apache.iotdb.commons.path.PartialPath;
-import org.apache.iotdb.commons.path.PathDeserializeUtil;
-import org.apache.iotdb.commons.path.PathPatternTree;
-import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
-import org.apache.iotdb.commons.schema.table.TsTable;
-import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
-import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
-import org.apache.iotdb.commons.service.metric.MetricService;
-import org.apache.iotdb.commons.trigger.TriggerInformation;
-import org.apache.iotdb.commons.utils.StatusUtils;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
-import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
-import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan;
-import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan;
-import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
-import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
-import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
-import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
-import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
-import org.apache.iotdb.confignode.manager.partition.PartitionManager;
-import org.apache.iotdb.confignode.persistence.ProcedureInfo;
-import org.apache.iotdb.confignode.procedure.PartitionTableAutoCleaner;
-import org.apache.iotdb.confignode.procedure.Procedure;
-import org.apache.iotdb.confignode.procedure.ProcedureExecutor;
-import org.apache.iotdb.confignode.procedure.ProcedureMetrics;
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler;
-import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler;
-import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
-import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
-import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
-import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure;
-import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
-import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleLeaderChangeProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeMetaSyncProcedure;
-import org.apache.iotdb.confignode.procedure.impl.pipe.task.AlterPipeProcedureV2;
-import org.apache.iotdb.confignode.procedure.impl.pipe.task.CreatePipeProcedureV2;
-import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2;
-import org.apache.iotdb.confignode.procedure.impl.pipe.task.StartPipeProcedureV2;
-import org.apache.iotdb.confignode.procedure.impl.pipe.task.StopPipeProcedureV2;
-import org.apache.iotdb.confignode.procedure.impl.region.AddRegionPeerProcedure;
-import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure;
-import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure;
-import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
-import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan;
-import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure;
-import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.AddViewColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.CreateTableViewProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewColumnProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewProcedure;
-import org.apache.iotdb.confignode.procedure.impl.schema.table.view.SetViewPropertiesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.CreateSubscriptionProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.DropSubscriptionProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.topic.CreateTopicProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.topic.DropTopicProcedure;
-import org.apache.iotdb.confignode.procedure.impl.subscription.topic.runtime.TopicMetaSyncProcedure;
-import org.apache.iotdb.confignode.procedure.impl.sync.AuthOperationProcedure;
-import org.apache.iotdb.confignode.procedure.impl.testonly.AddNeverFinishSubProcedureProcedure;
-import org.apache.iotdb.confignode.procedure.impl.testonly.CreateManyDatabasesProcedure;
-import org.apache.iotdb.confignode.procedure.impl.trigger.CreateTriggerProcedure;
-import org.apache.iotdb.confignode.procedure.impl.trigger.DropTriggerProcedure;
-import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
-import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore;
-import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-import org.apache.iotdb.confignode.procedure.store.ProcedureFactory;
-import org.apache.iotdb.confignode.procedure.store.ProcedureType;
-import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
-import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
-import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
-import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
-import org.apache.iotdb.confignode.rpc.thrift.TCreateTopicReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
-import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
-import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
-import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
-import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq;
-import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq;
-import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq;
-import org.apache.iotdb.consensus.ConsensusFactory;
-import org.apache.iotdb.db.exception.BatchProcessException;
-import org.apache.iotdb.db.schemaengine.template.Template;
-import org.apache.iotdb.db.utils.constant.SqlConstant;
-import org.apache.iotdb.rpc.RpcUtils;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import org.apache.ratis.util.AutoCloseableLock;
-import org.apache.tsfile.utils.Binary;
-import org.apache.tsfile.utils.Pair;
-import org.apache.tsfile.utils.ReadWriteIOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nonnull;
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.function.BiFunction;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
-
-public class ProcedureManager {
- private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureManager.class);
-
- private static final ConfigNodeConfig CONFIG_NODE_CONFIG =
- ConfigNodeDescriptor.getInstance().getConf();
- private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
-
- public static final long PROCEDURE_WAIT_TIME_OUT = COMMON_CONFIG.getDnConnectionTimeoutInMS();
- public static final int PROCEDURE_WAIT_RETRY_TIMEOUT = 10;
- private static final String PROCEDURE_TIMEOUT_MESSAGE =
- "Timed out to wait for procedure return. The procedure is still running.";
-
- private final ConfigManager configManager;
- private ProcedureExecutor<ConfigNodeProcedureEnv> executor;
- private ProcedureScheduler scheduler;
- private IProcedureStore store;
- private ConfigNodeProcedureEnv env;
-
- private final long planSizeLimit;
- private ProcedureMetrics procedureMetrics;
-
- private final PartitionTableAutoCleaner partitionTableCleaner;
-
- private final ReentrantLock tableLock = new ReentrantLock();
-
- public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo) {
- this.configManager = configManager;
- this.scheduler = new SimpleProcedureScheduler();
- this.store = new ConfigProcedureStore(configManager, procedureInfo);
- this.env = new ConfigNodeProcedureEnv(configManager, scheduler);
- this.executor = new ProcedureExecutor<>(env, store, scheduler);
- this.planSizeLimit =
- ConfigNodeDescriptor.getInstance()
- .getConf()
- .getConfigNodeRatisConsensusLogAppenderBufferSize()
- - IoTDBConstant.RAFT_LOG_BASIC_SIZE;
- this.procedureMetrics = new ProcedureMetrics(this);
- this.partitionTableCleaner = new PartitionTableAutoCleaner<>(configManager);
- }
-
- public void startExecutor() {
- if (!executor.isRunning()) {
- executor.init(CONFIG_NODE_CONFIG.getProcedureCoreWorkerThreadsCount());
- executor.startWorkers();
- executor.startCompletedCleaner(
- CONFIG_NODE_CONFIG.getProcedureCompletedCleanInterval(),
- CONFIG_NODE_CONFIG.getProcedureCompletedEvictTTL());
- executor.addInternalProcedure(partitionTableCleaner);
- store.start();
- LOGGER.info("ProcedureManager is started successfully.");
- }
- }
-
- public void stopExecutor() {
- if (executor.isRunning()) {
- executor.stop();
- if (!executor.isRunning()) {
- executor.join();
- store.stop();
- LOGGER.info("ProcedureManager is stopped successfully.");
- }
- executor.removeInternalProcedure(partitionTableCleaner);
- }
- }
-
- @TestOnly
- public TSStatus createManyDatabases() {
- this.executor.submitProcedure(new CreateManyDatabasesProcedure());
- return StatusUtils.OK;
- }
-
- @TestOnly
- public TSStatus testSubProcedure() {
- this.executor.submitProcedure(new AddNeverFinishSubProcedureProcedure());
- return StatusUtils.OK;
- }
-
- public TSStatus deleteDatabases(
- final List<TDatabaseSchema> deleteSgSchemaList, final boolean isGeneratedByPipe) {
- final List<DeleteDatabaseProcedure> procedures = new ArrayList<>();
- final long startCheckTimeForProcedures = System.currentTimeMillis();
- for (final TDatabaseSchema databaseSchema : deleteSgSchemaList) {
- final String database = databaseSchema.getName();
- boolean hasOverlappedTask = false;
- synchronized (this) {
- while (executor.isRunning()
- && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) {
- final Pair<Long, Boolean> procedureIdDuplicatePair =
- checkDuplicateTableTask(
- database, null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE);
- hasOverlappedTask = procedureIdDuplicatePair.getRight();
-
- if (Boolean.FALSE.equals(procedureIdDuplicatePair.getRight())) {
- DeleteDatabaseProcedure procedure =
- new DeleteDatabaseProcedure(databaseSchema, isGeneratedByPipe);
- this.executor.submitProcedure(procedure);
- procedures.add(procedure);
- break;
- }
- try {
- wait(PROCEDURE_WAIT_RETRY_TIMEOUT);
- } catch (final InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- String.format(
- "Some other task is operating table under the database %s, please retry after the procedure finishes.",
- database));
- }
- }
- }
- List<TSStatus> results = new ArrayList<>(procedures.size());
- procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure)));
- // Clear the previously deleted regions
- final PartitionManager partitionManager = getConfigManager().getPartitionManager();
- partitionManager.getRegionMaintainer().submit(partitionManager::maintainRegionReplicas);
- if (results.stream()
- .allMatch(result -> result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode())) {
- return StatusUtils.OK;
- } else {
- return RpcUtils.getStatus(results);
- }
- }
-
- public TSStatus deleteTimeSeries(
- String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) {
- DeleteTimeSeriesProcedure procedure = null;
- synchronized (this) {
- boolean hasOverlappedTask = false;
- ProcedureType type;
- DeleteTimeSeriesProcedure deleteTimeSeriesProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.DELETE_TIMESERIES_PROCEDURE)) {
- continue;
- }
- deleteTimeSeriesProcedure = ((DeleteTimeSeriesProcedure) runningProcedure);
- if (queryId.equals(deleteTimeSeriesProcedure.getQueryId())) {
- procedure = deleteTimeSeriesProcedure;
- break;
- }
- if (patternTree.isOverlapWith(deleteTimeSeriesProcedure.getPatternTree())) {
- hasOverlappedTask = true;
- break;
- }
- }
-
- if (procedure == null) {
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is deleting some target timeseries.");
- }
- procedure = new DeleteTimeSeriesProcedure(queryId, patternTree, isGeneratedByPipe);
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- public TSStatus deleteLogicalView(TDeleteLogicalViewReq req) {
- String queryId = req.getQueryId();
- PathPatternTree patternTree =
- PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
- DeleteLogicalViewProcedure procedure = null;
- synchronized (this) {
- boolean hasOverlappedTask = false;
- ProcedureType type;
- DeleteLogicalViewProcedure deleteLogicalViewProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.DELETE_LOGICAL_VIEW_PROCEDURE)) {
- continue;
- }
- deleteLogicalViewProcedure = ((DeleteLogicalViewProcedure) runningProcedure);
- if (queryId.equals(deleteLogicalViewProcedure.getQueryId())) {
- procedure = deleteLogicalViewProcedure;
- break;
- }
- if (patternTree.isOverlapWith(deleteLogicalViewProcedure.getPatternTree())) {
- hasOverlappedTask = true;
- break;
- }
- }
-
- if (procedure == null) {
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is deleting some target views.");
- }
- procedure =
- new DeleteLogicalViewProcedure(
- queryId, patternTree, req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- public TSStatus alterLogicalView(final TAlterLogicalViewReq req) {
- final String queryId = req.getQueryId();
- final ByteBuffer byteBuffer = ByteBuffer.wrap(req.getViewBinary());
- final Map<PartialPath, ViewExpression> viewPathToSourceMap = new HashMap<>();
- final int size = byteBuffer.getInt();
- PartialPath path;
- ViewExpression viewExpression;
- for (int i = 0; i < size; i++) {
- path = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer);
- viewExpression = ViewExpression.deserialize(byteBuffer);
- viewPathToSourceMap.put(path, viewExpression);
- }
-
- AlterLogicalViewProcedure procedure = null;
- synchronized (this) {
- ProcedureType type;
- AlterLogicalViewProcedure alterLogicalViewProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.ALTER_LOGICAL_VIEW_PROCEDURE)) {
- continue;
- }
- alterLogicalViewProcedure = ((AlterLogicalViewProcedure) runningProcedure);
- if (queryId.equals(alterLogicalViewProcedure.getQueryId())) {
- procedure = alterLogicalViewProcedure;
- break;
- }
- }
-
- if (procedure == null) {
- procedure =
- new AlterLogicalViewProcedure(
- queryId,
- viewPathToSourceMap,
- req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- public TSStatus setSchemaTemplate(
- String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) {
- SetTemplateProcedure procedure = null;
- synchronized (this) {
- boolean hasOverlappedTask = false;
- ProcedureType type;
- SetTemplateProcedure setTemplateProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
- continue;
- }
- setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
- if (queryId.equals(setTemplateProcedure.getQueryId())) {
- procedure = setTemplateProcedure;
- break;
- }
- if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
- hasOverlappedTask = true;
- break;
- }
- }
-
- if (procedure == null) {
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is setting template on target path.");
- }
- procedure =
- new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe);
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- public TSStatus deactivateTemplate(
- String queryId, Map<PartialPath, List<Template>> templateSetInfo, boolean isGeneratedByPipe) {
- DeactivateTemplateProcedure procedure = null;
- synchronized (this) {
- boolean hasOverlappedTask = false;
- ProcedureType type;
- DeactivateTemplateProcedure deactivateTemplateProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.DEACTIVATE_TEMPLATE_PROCEDURE)) {
- continue;
- }
- deactivateTemplateProcedure = (DeactivateTemplateProcedure) runningProcedure;
- if (queryId.equals(deactivateTemplateProcedure.getQueryId())) {
- procedure = deactivateTemplateProcedure;
- break;
- }
- for (PartialPath pattern : templateSetInfo.keySet()) {
- for (PartialPath existingPattern :
- deactivateTemplateProcedure.getTemplateSetInfo().keySet()) {
- if (pattern.overlapWith(existingPattern)) {
- hasOverlappedTask = true;
- break;
- }
- }
- if (hasOverlappedTask) {
- break;
- }
- }
- if (hasOverlappedTask) {
- break;
- }
- }
-
- if (procedure == null) {
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is deactivating some target template from target path.");
- }
- procedure = new DeactivateTemplateProcedure(queryId, templateSetInfo, isGeneratedByPipe);
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- public TSStatus unsetSchemaTemplate(
- String queryId, Template template, PartialPath path, boolean isGeneratedByPipe) {
- UnsetTemplateProcedure procedure = null;
- synchronized (this) {
- boolean hasOverlappedTask = false;
- ProcedureType type;
- UnsetTemplateProcedure unsetTemplateProcedure;
- for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
- type = ProcedureFactory.getProcedureType(runningProcedure);
- if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
- continue;
- }
- unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
- if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
- procedure = unsetTemplateProcedure;
- break;
- }
- if (template.getId() == unsetTemplateProcedure.getTemplateId()
- && path.equals(unsetTemplateProcedure.getPath())) {
- hasOverlappedTask = true;
- break;
- }
- }
-
- if (procedure == null) {
- if (hasOverlappedTask) {
- return RpcUtils.getStatus(
- TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
- "Some other task is unsetting target template from target path "
- + path.getFullPath());
- }
- procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe);
- this.executor.submitProcedure(procedure);
- }
- }
- return waitingProcedureFinished(procedure);
- }
-
- /**
- * Generate an {@link AddConfigNodeProcedure}, and serially execute all the {@link
- * AddConfigNodeProcedure}s.
- */
- public void addConfigNode(TConfigNodeRegisterReq req) {
- final AddConfigNodeProcedure addConfigNodeProcedure =
- new AddConfigNodeProcedure(req.getConfigNodeLocation(), req.getVersionInfo());
- this.executor.submitProcedure(addConfigNodeProcedure);
- }
-
- /**
- * Generate a {@link RemoveConfigNodeProcedure}, and serially execute all the {@link
- * RemoveConfigNodeProcedure}s.
- */
- public void removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) {
- final RemoveConfigNodeProcedure removeConfigNodeProcedure =
- new RemoveConfigNodeProcedure(removeConfigNodePlan.getConfigNodeLocation());
- this.executor.submitProcedure(removeConfigNodeProcedure);
- LOGGER.info("Submit RemoveConfigNodeProcedure successfully: {}", removeConfigNodePlan);
- }
-
- /**
- * Generate {@link RemoveDataNodesProcedure}s, and serially execute all the {@link
- * RemoveDataNodesProcedure}s.
- */
- public boolean removeDataNode(RemoveDataNodePlan removeDataNodePlan) {
- Map<Integer, NodeStatus> nodeStatusMap = new HashMap<>();
- removeDataNodePlan
- .getDataNodeLocations()
- .forEach(
- datanode ->
- nodeStatusMap.put(
- datanode.getDataNodeId(),
- configManager.getLoadManager().getNodeStatus(datanode.getDataNodeId())));
- this.executor.submitProcedure(
- new RemoveDataNodesProcedure(removeDataNodePlan.getDataNodeLocations(), nodeStatusMap));
- LOGGER.info(
- "Submit RemoveDataNodesProcedure successfully, {}",
- removeDataNodePlan.getDataNodeLocations());
- return true;
- }
-
- public boolean removeAINode(RemoveAINodePlan removeAINodePlan) {
- this.executor.submitProcedure(new RemoveAINodeProcedure(removeAINodePlan.getAINodeLocation()));
- LOGGER.info(
- "Submit RemoveAINodeProcedure successfully, {}", removeAINodePlan.getAINodeLocation());
- return true;
- }
-
- public TSStatus checkRemoveDataNodes(List<TDataNodeLocation> dataNodeLocations) {
- // 1. Only one RemoveDataNodesProcedure is allowed in the cluster
- Optional<Procedure<ConfigNodeProcedureEnv>> anotherRemoveProcedure =
- getExecutor().getProcedures().values().stream()
- .filter(
- procedure -> {
- if (procedure instanceof RemoveDataNodesProcedure) {
- return !procedure.isFinished();
- }
- return false;
- })
- .findAny();
-
- String failMessage = null;
- if (anotherRemoveProcedure.isPresent()) {
- List<TDataNodeLocation> anotherRemoveDataNodes =
- ((RemoveDataNodesProcedure) anotherRemoveProcedure.get()).getRemovedDataNodes();
- failMessage =
- String.format(
- "Submit RemoveDataNodesProcedure failed, "
- + "because another RemoveDataNodesProcedure %s is already in processing. "
- + "IoTDB is able to have at most 1 RemoveDataNodesProcedure at the same time. "
- + "For further information, please search [pid%d] in log. ",
- anotherRemoveDataNodes, anotherRemoveProcedure.get().getProcId());
- }
-
- // 2. Check if the RemoveDataNodesProcedure conflicts with the RegionMigrateProcedure
- Set<TConsensusGroupId> removedDataNodesRegionSet =
- getEnv().getRemoveDataNodeHandler().getRemovedDataNodesRegionSet(dataNodeLocations);
- Optional<Procedure<ConfigNodeProcedureEnv>> conflictRegionMigrateProcedure =
- getExecutor().getProcedures().values().stream()
- .filter(
- procedure -> {
- if (procedure instanceof RegionMigrateProcedure) {
- RegionMigrateProcedure regionMigrateProcedure =
- (RegionMigrateProcedure) procedure;
- if (regionMigrateProcedure.isFinished()) {
- return false;
- }
- return removedDataNodesRegionSet.contains(regionMigrateProcedure.getRegionId())
- || dataNodeLocations.contains(regionMigrateProcedure.getDestDataNode());
- }
- return false;
- })
- .findAny();
- if (conflictRegionMigrateProcedure.isPresent()) {
- failMessage =
- String.format(
- "Submit RemoveDataNodesProcedure failed, "
- + "because another RegionMigrateProcedure %s is already in processing which conflicts with this RemoveDataNodesProcedure. "
- + "The RegionMigrateProcedure is migrating the region %s to the DataNode %s. "
- + "For further information, please search [pid%d] in log. ",
- conflictRegionMigrateProcedure.get().getProcId(),
- ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getRegionId(),
- ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getDestDataNode(),
- conflictRegionMigrateProcedure.get().getProcId());
- }
- // 3. Check if the RegionMigrateProcedure generated by RemoveDataNodesProcedure conflicts with
- // each other
- List<RegionMigrationPlan> regionMigrationPlans =
- getEnv().getRemoveDataNodeHandler().getRegionMigrationPlans(dataNodeLocations);
- removedDataNodesRegionSet.clear();
- for (RegionMigrationPlan regionMigrationPlan : regionMigrationPlans) {
- if (removedDataNodesRegionSet.contains(regionMigrationPlan.getRegionId())) {
- failMessage =
- String.format(
- "Submit RemoveDataNodesProcedure failed, "
- + "because the RegionMigrateProcedure generated by this RemoveDataNodesProcedure conflicts with each other. "
- + "Only one replica of the same consensus group is allowed to be migrated at the same time."
- + "The conflict region id is %s . ",
- regionMigrationPlan.getRegionId());
- break;
- }
- removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId());
- }
-
- // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are
- // not the remove DataNodes
-
- for (TDataNodeLocation removeDataNode : dataNodeLocations) {
- Set<TDataNodeLocation> relatedDataNodes =
- getEnv().getRemoveDataNodeHandler().getRelatedDataNodeLocations(removeDataNode);
- relatedDataNodes.remove(removeDataNode);
-
- for (TDataNodeLocation relatedDataNode : relatedDataNodes) {
- NodeStatus nodeStatus =
- getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId());
- if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) {
- failMessage =
- String.format(
- "Submit RemoveDataNodesProcedure failed, "
- + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, "
- + "the remove operation cannot be performed for security reasons. "
- + "Please check the status of the node %s and ensure it is running.",
- relatedDataNode.getDataNodeId());
- }
- }
- }
-
- if (failMessage != null) {
- LOGGER.warn(failMessage);
- TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_DATANODE_ERROR.getStatusCode());
- failStatus.setMessage(failMessage);
- return failStatus;
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- // region region operation related check
-
- /**
- * Checks whether region migration is allowed.
- *
- * @param migrateRegionReq the migration request details
- * @param regionGroupId the ID of the consensus group for the region
- * @param originalDataNode the original DataNode location from which the region is being migrated
- * @param destDataNode the destination DataNode location to which the region is being migrated
- * @param coordinatorForAddPeer the DataNode location acting as the coordinator for adding a peer
- * @return the status of the migration request (TSStatus)
- */
- private TSStatus checkMigrateRegion(
- TMigrateRegionReq migrateRegionReq,
- TConsensusGroupId regionGroupId,
- TDataNodeLocation originalDataNode,
- TDataNodeLocation destDataNode,
- TDataNodeLocation coordinatorForAddPeer) {
- String failMessage;
- if ((failMessage =
- regionOperationCommonCheck(
- regionGroupId,
- destDataNode,
- Arrays.asList(
- new Pair<>("Original DataNode", originalDataNode),
- new Pair<>("Destination DataNode", destDataNode),
- new Pair<>("Coordinator for add peer", coordinatorForAddPeer)),
- migrateRegionReq.getModel()))
- != null) {
- // do nothing
- } else if (configManager
- .getPartitionManager()
- .getAllReplicaSets(originalDataNode.getDataNodeId())
- .stream()
- .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
- failMessage =
- String.format(
- "Submit RegionMigrateProcedure failed, because the original DataNode %s doesn't contain Region %s",
- migrateRegionReq.getFromId(), migrateRegionReq.getRegionId());
- } else if (configManager
- .getPartitionManager()
- .getAllReplicaSets(destDataNode.getDataNodeId())
- .stream()
- .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
- failMessage =
- String.format(
- "Submit RegionMigrateProcedure failed, because the target DataNode %s already contains Region %s",
- migrateRegionReq.getToId(), migrateRegionReq.getRegionId());
- }
-
- if (failMessage != null) {
- LOGGER.warn(failMessage);
- TSStatus failStatus = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- failStatus.setMessage(failMessage);
- return failStatus;
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- private TSStatus checkReconstructRegion(
- TReconstructRegionReq req,
- TConsensusGroupId regionId,
- TDataNodeLocation targetDataNode,
- TDataNodeLocation coordinator) {
- String failMessage =
- regionOperationCommonCheck(
- regionId,
- targetDataNode,
- Arrays.asList(
- new Pair<>("Target DataNode", targetDataNode),
- new Pair<>("Coordinator", coordinator)),
- req.getModel());
-
- ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
- if (configManager
- .getPartitionManager()
- .getAllReplicaSetsMap(regionId.getType())
- .get(regionId)
- .getDataNodeLocationsSize()
- == 1) {
- failMessage = String.format("%s only has 1 replica, it cannot be reconstructed", regionId);
- } else if (configManager
- .getPartitionManager()
- .getAllReplicaSets(targetDataNode.getDataNodeId())
- .stream()
- .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
- failMessage =
- String.format(
- "Submit ReconstructRegionProcedure failed, because the target DataNode %s doesn't contain Region %s",
- req.getDataNodeId(), regionId);
- }
-
- if (failMessage != null) {
- LOGGER.warn(failMessage);
- TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
- failStatus.setMessage(failMessage);
- return failStatus;
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- private TSStatus checkExtendRegion(
- TExtendRegionReq req,
- TConsensusGroupId regionId,
- TDataNodeLocation targetDataNode,
- TDataNodeLocation coordinator) {
- String failMessage =
- regionOperationCommonCheck(
- regionId,
- targetDataNode,
- Arrays.asList(
- new Pair<>("Target DataNode", targetDataNode),
- new Pair<>("Coordinator", coordinator)),
- req.getModel());
- if (configManager
- .getPartitionManager()
- .getAllReplicaSets(targetDataNode.getDataNodeId())
- .stream()
- .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
- failMessage =
- String.format(
- "Target DataNode %s already contains region %s",
- targetDataNode.getDataNodeId(), regionId);
- }
-
- if (failMessage != null) {
- LOGGER.warn(failMessage);
- TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
- failStatus.setMessage(failMessage);
- return failStatus;
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- private TSStatus checkRemoveRegion(
- TRemoveRegionReq req,
- TConsensusGroupId regionId,
- @Nullable TDataNodeLocation targetDataNode,
- TDataNodeLocation coordinator) {
- String failMessage =
- regionOperationCommonCheck(
- regionId,
- targetDataNode,
- Arrays.asList(new Pair<>("Coordinator", coordinator)),
- req.getModel());
-
- if (configManager
- .getPartitionManager()
- .getAllReplicaSetsMap(regionId.getType())
- .get(regionId)
- .getDataNodeLocationsSize()
- == 1) {
- failMessage = String.format("%s only has 1 replica, it cannot be removed", regionId);
- } else if (targetDataNode != null
- && configManager
- .getPartitionManager()
- .getAllReplicaSets(targetDataNode.getDataNodeId())
- .stream()
- .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
- failMessage =
- String.format(
- "Target DataNode %s doesn't contain Region %s", req.getDataNodeId(), regionId);
- }
-
- if (failMessage != null) {
- LOGGER.warn(failMessage);
- TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_REGION_PEER_ERROR.getStatusCode());
- failStatus.setMessage(failMessage);
- return failStatus;
- }
- return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- /**
- * The common checks of all region operations, include migration, reconstruction, extension,
- * removing
- *
- * @param regionId region group id, also called consensus group id
- * @param targetDataNode DataNode should in Running status
- * @param relatedDataNodes Pair<Identity, Node Location>
- * @return The reason if check failed, or null if check pass
- */
- private String regionOperationCommonCheck(
- TConsensusGroupId regionId,
- TDataNodeLocation targetDataNode,
- List<Pair<String, TDataNodeLocation>> relatedDataNodes,
- Model model) {
- String failMessage;
- ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
-
- if (TConsensusGroupType.DataRegion == regionId.getType()
- && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) {
- failMessage = "SimpleConsensus not supports region operation.";
- } else if (TConsensusGroupType.SchemaRegion == regionId.getType()
- && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getSchemaRegionConsensusProtocolClass())) {
- failMessage = "SimpleConsensus not supports region operation.";
- } else if ((failMessage = checkRegionOperationDuplication(regionId)) != null) {
- // need to do nothing more
- } else if (relatedDataNodes.stream().anyMatch(pair -> pair.getRight() == null)) {
- Pair<String, TDataNodeLocation> nullPair =
- relatedDataNodes.stream().filter(pair -> pair.getRight() == null).findAny().get();
- failMessage = String.format("Cannot find %s", nullPair.getLeft());
- } else if (targetDataNode != null
- && !configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running).stream()
- .map(TDataNodeConfiguration::getLocation)
- .map(TDataNodeLocation::getDataNodeId)
- .collect(Collectors.toSet())
- .contains(targetDataNode.getDataNodeId())) {
- // Here we only check Running DataNode to implement migration, because removing nodes may not
- // exist when add peer is performing
- failMessage =
- String.format(
- "Target DataNode %s is not in Running status.", targetDataNode.getDataNodeId());
- } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode))
- != null) {
- // need to do nothing more
- } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) {
- // need to do nothing more
- }
-
- return failMessage;
- }
-
- private String checkRegionOperationWithRemoveDataNode(
- TConsensusGroupId regionId, TDataNodeLocation targetDataNode) {
- Optional<Procedure<ConfigNodeProcedureEnv>> conflictRemoveDataNodesProcedure =
- getExecutor().getProcedures().values().stream()
- .filter(
- procedure -> {
- if (procedure instanceof RemoveDataNodesProcedure) {
- return !procedure.isFinished();
- }
- return false;
- })
- .findAny();
-
- if (conflictRemoveDataNodesProcedure.isPresent()) {
- RemoveDataNodeHandler removeDataNodeHandler = env.getRemoveDataNodeHandler();
- List<TDataNodeLocation> removedDataNodes =
- ((RemoveDataNodesProcedure) conflictRemoveDataNodesProcedure.get()).getRemovedDataNodes();
- Set<TConsensusGroupId> removedDataNodesRegionSet =
- removeDataNodeHandler.getRemovedDataNodesRegionSet(removedDataNodes);
- if (removedDataNodesRegionSet.contains(regionId)) {
- return String.format(
- "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
- + "The RemoveDataNodesProcedure is removing the DataNodes %s which contains the region %s. "
- + "For further information, please search [pid%d] in log. ",
- conflictRemoveDataNodesProcedure.get().getProcId(),
- removedDataNodes,
- regionId,
- conflictRemoveDataNodesProcedure.get().getProcId());
- } else if (removedDataNodes.contains(targetDataNode)) {
- return String.format(
- "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
- + "The RemoveDataNodesProcedure is removing the target DataNode %s. "
- + "For further information, please search [pid%d] in log. ",
- conflictRemoveDataNodesProcedure.get().getProcId(),
- targetDataNode,
- conflictRemoveDataNodesProcedure.get().getProcId());
- }
- }
- return null;
- }
-
- private String checkRegionOperationDuplication(TConsensusGroupId regionId) {
- List<? extends RegionOperationProcedure<?>> otherRegionMemberChangeProcedures =
- getRegionOperationProcedures()
- .filter(
- regionMemberChangeProcedure ->
- regionId.equals(regionMemberChangeProcedure.getRegionId()))
- .collect(Collectors.toList());
- if (!otherRegionMemberChangeProcedures.isEmpty()) {
- return String.format(
- "%s has some other region operation procedures in progress, their procedure id is: %s",
- regionId, otherRegionMemberChangeProcedures);
- }
- return null;
- }
-
- public List<TConsensusGroupId> getRegionOperationConsensusIds() {
- return getRegionOperationProcedures()
- .map(RegionOperationProcedure::getRegionId)
- .distinct()
- .collect(Collectors.toList());
- }
-
- private Stream<RegionOperationProcedure<?>> getRegionOperationProcedures() {
- return getExecutor().getProcedures().values().stream()
- .filter(procedure -> !procedure.isFinished())
- .filter(procedure -> procedure instanceof RegionOperationProcedure)
- .map(procedure -> (RegionOperationProcedure<?>) procedure);
- }
-
- private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) {
- String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId);
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.manager;
+1
+1import org.apache.iotdb.common.rpc.thrift.Model;
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+1import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+1import org.apache.iotdb.common.rpc.thrift.TSStatus;
+1import org.apache.iotdb.commons.cluster.NodeStatus;
+1import org.apache.iotdb.commons.conf.CommonConfig;
+1import org.apache.iotdb.commons.conf.CommonDescriptor;
+1import org.apache.iotdb.commons.conf.IoTDBConstant;
+1import org.apache.iotdb.commons.exception.IoTDBException;
+1import org.apache.iotdb.commons.path.PartialPath;
+1import org.apache.iotdb.commons.path.PathDeserializeUtil;
+1import org.apache.iotdb.commons.path.PathPatternTree;
+1import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
+1import org.apache.iotdb.commons.schema.table.TsTable;
+1import org.apache.iotdb.commons.schema.table.column.TsTableColumnSchemaUtil;
+1import org.apache.iotdb.commons.schema.view.viewExpression.ViewExpression;
+1import org.apache.iotdb.commons.service.metric.MetricService;
+1import org.apache.iotdb.commons.trigger.TriggerInformation;
+1import org.apache.iotdb.commons.utils.StatusUtils;
+1import org.apache.iotdb.commons.utils.TestOnly;
+1import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+1import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+1import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan;
+1import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.database.SetTTLPlan;
+1import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
+1import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+1import org.apache.iotdb.confignode.persistence.ProcedureInfo;
+1import org.apache.iotdb.confignode.procedure.PartitionTableAutoCleaner;
+1import org.apache.iotdb.confignode.procedure.Procedure;
+1import org.apache.iotdb.confignode.procedure.ProcedureExecutor;
+1import org.apache.iotdb.confignode.procedure.ProcedureMetrics;
+1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+1import org.apache.iotdb.confignode.procedure.env.RegionMaintainHandler;
+1import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler;
+1import org.apache.iotdb.confignode.procedure.impl.cq.CreateCQProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.model.CreateModelProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.model.DropModelProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.node.AddConfigNodeProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.node.RemoveAINodeProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.node.RemoveConfigNodeProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.node.RemoveDataNodesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.CreatePipePluginProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.plugin.DropPipePluginProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleLeaderChangeProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeHandleMetaChangeProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.runtime.PipeMetaSyncProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.task.AlterPipeProcedureV2;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.task.CreatePipeProcedureV2;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.task.DropPipeProcedureV2;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.task.StartPipeProcedureV2;
+1import org.apache.iotdb.confignode.procedure.impl.pipe.task.StopPipeProcedureV2;
+1import org.apache.iotdb.confignode.procedure.impl.region.AddRegionPeerProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.region.CreateRegionGroupsProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.region.ReconstructRegionProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrateProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.region.RegionMigrationPlan;
+1import org.apache.iotdb.confignode.procedure.impl.region.RegionOperationProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.region.RemoveRegionPeerProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.AlterLogicalViewProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.DeactivateTemplateProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteDatabaseProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteLogicalViewProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.DeleteTimeSeriesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.SetTTLProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.SetTemplateProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.UnsetTemplateProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.AbstractAlterOrDropTableProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.AddTableColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.CreateTableProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.DeleteDevicesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.DropTableProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.RenameTableProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.SetTablePropertiesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.AddViewColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.CreateTableViewProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.DropViewProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewColumnProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.RenameViewProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.schema.table.view.SetViewPropertiesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.CreateConsumerProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.DropConsumerProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.consumer.runtime.ConsumerGroupMetaSyncProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.CreateSubscriptionProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.subscription.DropSubscriptionProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.CreateTopicProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.DropTopicProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.subscription.topic.runtime.TopicMetaSyncProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.sync.AuthOperationProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.testonly.AddNeverFinishSubProcedureProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.testonly.CreateManyDatabasesProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.trigger.CreateTriggerProcedure;
+1import org.apache.iotdb.confignode.procedure.impl.trigger.DropTriggerProcedure;
+1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+1import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
+1import org.apache.iotdb.confignode.procedure.store.ConfigProcedureStore;
+1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+1import org.apache.iotdb.confignode.procedure.store.ProcedureFactory;
+1import org.apache.iotdb.confignode.procedure.store.ProcedureType;
+1import org.apache.iotdb.confignode.rpc.thrift.TAlterLogicalViewReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TAlterOrDropTableReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TCloseConsumerReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeRegisterReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TCreateCQReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TCreateConsumerReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TCreateTopicReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
+1import org.apache.iotdb.confignode.rpc.thrift.TDeleteLogicalViewReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TDeleteTableDeviceResp;
+1import org.apache.iotdb.confignode.rpc.thrift.TDropPipePluginReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TExtendRegionReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TMigrateRegionReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TReconstructRegionReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TRemoveRegionReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq;
+1import org.apache.iotdb.consensus.ConsensusFactory;
+1import org.apache.iotdb.db.exception.BatchProcessException;
+1import org.apache.iotdb.db.schemaengine.template.Template;
+1import org.apache.iotdb.db.utils.constant.SqlConstant;
+1import org.apache.iotdb.rpc.RpcUtils;
+1import org.apache.iotdb.rpc.TSStatusCode;
+1
+1import org.apache.ratis.util.AutoCloseableLock;
+1import org.apache.tsfile.utils.Binary;
+1import org.apache.tsfile.utils.Pair;
+1import org.apache.tsfile.utils.ReadWriteIOUtils;
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import javax.annotation.Nonnull;
+1import javax.annotation.Nullable;
+1
+1import java.io.IOException;
+1import java.nio.ByteBuffer;
+1import java.util.ArrayList;
+1import java.util.Arrays;
+1import java.util.HashMap;
+1import java.util.List;
+1import java.util.Map;
+1import java.util.Objects;
+1import java.util.Optional;
+1import java.util.Set;
+1import java.util.concurrent.ScheduledExecutorService;
+1import java.util.concurrent.locks.ReentrantLock;
+1import java.util.function.BiFunction;
+1import java.util.stream.Collectors;
+1import java.util.stream.Stream;
+1
+1public class ProcedureManager {
+1 private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureManager.class);
+1
+1 private static final ConfigNodeConfig CONFIG_NODE_CONFIG =
+1 ConfigNodeDescriptor.getInstance().getConf();
+1 private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+1
+1 public static final long PROCEDURE_WAIT_TIME_OUT = COMMON_CONFIG.getDnConnectionTimeoutInMS();
+1 public static final int PROCEDURE_WAIT_RETRY_TIMEOUT = 10;
+1 private static final String PROCEDURE_TIMEOUT_MESSAGE =
+1 "Timed out to wait for procedure return. The procedure is still running.";
+1
+1 private final ConfigManager configManager;
+1 private ProcedureExecutor<ConfigNodeProcedureEnv> executor;
+1 private ProcedureScheduler scheduler;
+1 private IProcedureStore store;
+1 private ConfigNodeProcedureEnv env;
+1
+1 private final long planSizeLimit;
+1 private ProcedureMetrics procedureMetrics;
+1
+1 private final PartitionTableAutoCleaner partitionTableCleaner;
+1
+1 private final ReentrantLock tableLock = new ReentrantLock();
+1
+1 public ProcedureManager(ConfigManager configManager, ProcedureInfo procedureInfo) {
+1 this.configManager = configManager;
+1 this.scheduler = new SimpleProcedureScheduler();
+1 this.store = new ConfigProcedureStore(configManager, procedureInfo);
+1 this.env = new ConfigNodeProcedureEnv(configManager, scheduler);
+1 this.executor = new ProcedureExecutor<>(env, store, scheduler);
+1 this.planSizeLimit =
+1 ConfigNodeDescriptor.getInstance()
+1 .getConf()
+1 .getConfigNodeRatisConsensusLogAppenderBufferSize()
+1 - IoTDBConstant.RAFT_LOG_BASIC_SIZE;
+1 this.procedureMetrics = new ProcedureMetrics(this);
+1 this.partitionTableCleaner = new PartitionTableAutoCleaner<>(configManager);
+1 }
+1
+1 public void startExecutor() {
+1 if (!executor.isRunning()) {
+1 executor.init(CONFIG_NODE_CONFIG.getProcedureCoreWorkerThreadsCount());
+1 executor.startWorkers();
+1 executor.startCompletedCleaner(
+1 CONFIG_NODE_CONFIG.getProcedureCompletedCleanInterval(),
+1 CONFIG_NODE_CONFIG.getProcedureCompletedEvictTTL());
+1 executor.addInternalProcedure(partitionTableCleaner);
+1 store.start();
+1 LOGGER.info("ProcedureManager is started successfully.");
+1 }
+1 }
+1
+1 public void stopExecutor() {
+1 if (executor.isRunning()) {
+1 executor.stop();
+1 if (!executor.isRunning()) {
+1 executor.join();
+1 store.stop();
+1 LOGGER.info("ProcedureManager is stopped successfully.");
+1 }
+1 executor.removeInternalProcedure(partitionTableCleaner);
+1 }
+1 }
+1
+1 @TestOnly
+1 public TSStatus createManyDatabases() {
+1 this.executor.submitProcedure(new CreateManyDatabasesProcedure());
+1 return StatusUtils.OK;
+1 }
+1
+1 @TestOnly
+1 public TSStatus testSubProcedure() {
+1 this.executor.submitProcedure(new AddNeverFinishSubProcedureProcedure());
+1 return StatusUtils.OK;
+1 }
+1
+1 public TSStatus deleteDatabases(
+1 final List<TDatabaseSchema> deleteSgSchemaList, final boolean isGeneratedByPipe) {
+1 final List<DeleteDatabaseProcedure> procedures = new ArrayList<>();
+1 final long startCheckTimeForProcedures = System.currentTimeMillis();
+1 for (final TDatabaseSchema databaseSchema : deleteSgSchemaList) {
+1 final String database = databaseSchema.getName();
+1 boolean hasOverlappedTask = false;
+1 synchronized (this) {
+1 while (executor.isRunning()
+1 && System.currentTimeMillis() - startCheckTimeForProcedures < PROCEDURE_WAIT_TIME_OUT) {
+1 final Pair<Long, Boolean> procedureIdDuplicatePair =
+1 checkDuplicateTableTask(
+1 database, null, null, null, null, ProcedureType.DELETE_DATABASE_PROCEDURE);
+1 hasOverlappedTask = procedureIdDuplicatePair.getRight();
+1
+1 if (Boolean.FALSE.equals(procedureIdDuplicatePair.getRight())) {
+1 DeleteDatabaseProcedure procedure =
+1 new DeleteDatabaseProcedure(databaseSchema, isGeneratedByPipe);
+1 this.executor.submitProcedure(procedure);
+1 procedures.add(procedure);
+1 break;
+1 }
+1 try {
+1 wait(PROCEDURE_WAIT_RETRY_TIMEOUT);
+1 } catch (final InterruptedException e) {
+1 Thread.currentThread().interrupt();
+1 }
+1 }
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 String.format(
+1 "Some other task is operating table under the database %s, please retry after the procedure finishes.",
+1 database));
+1 }
+1 }
+1 }
+1 List<TSStatus> results = new ArrayList<>(procedures.size());
+1 procedures.forEach(procedure -> results.add(waitingProcedureFinished(procedure)));
+1 // Clear the previously deleted regions
+1 final PartitionManager partitionManager = getConfigManager().getPartitionManager();
+1 partitionManager.getRegionMaintainer().submit(partitionManager::maintainRegionReplicas);
+1 if (results.stream()
+1 .allMatch(result -> result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode())) {
+1 return StatusUtils.OK;
+1 } else {
+1 return RpcUtils.getStatus(results);
+1 }
+1 }
+1
+1 public TSStatus deleteTimeSeries(
+1 String queryId, PathPatternTree patternTree, boolean isGeneratedByPipe) {
+1 DeleteTimeSeriesProcedure procedure = null;
+1 synchronized (this) {
+1 boolean hasOverlappedTask = false;
+1 ProcedureType type;
+1 DeleteTimeSeriesProcedure deleteTimeSeriesProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.DELETE_TIMESERIES_PROCEDURE)) {
+1 continue;
+1 }
+1 deleteTimeSeriesProcedure = ((DeleteTimeSeriesProcedure) runningProcedure);
+1 if (queryId.equals(deleteTimeSeriesProcedure.getQueryId())) {
+1 procedure = deleteTimeSeriesProcedure;
+1 break;
+1 }
+1 if (patternTree.isOverlapWith(deleteTimeSeriesProcedure.getPatternTree())) {
+1 hasOverlappedTask = true;
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 "Some other task is deleting some target timeseries.");
+1 }
+1 procedure = new DeleteTimeSeriesProcedure(queryId, patternTree, isGeneratedByPipe);
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 public TSStatus deleteLogicalView(TDeleteLogicalViewReq req) {
+1 String queryId = req.getQueryId();
+1 PathPatternTree patternTree =
+1 PathPatternTree.deserialize(ByteBuffer.wrap(req.getPathPatternTree()));
+1 DeleteLogicalViewProcedure procedure = null;
+1 synchronized (this) {
+1 boolean hasOverlappedTask = false;
+1 ProcedureType type;
+1 DeleteLogicalViewProcedure deleteLogicalViewProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.DELETE_LOGICAL_VIEW_PROCEDURE)) {
+1 continue;
+1 }
+1 deleteLogicalViewProcedure = ((DeleteLogicalViewProcedure) runningProcedure);
+1 if (queryId.equals(deleteLogicalViewProcedure.getQueryId())) {
+1 procedure = deleteLogicalViewProcedure;
+1 break;
+1 }
+1 if (patternTree.isOverlapWith(deleteLogicalViewProcedure.getPatternTree())) {
+1 hasOverlappedTask = true;
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 "Some other task is deleting some target views.");
+1 }
+1 procedure =
+1 new DeleteLogicalViewProcedure(
+1 queryId, patternTree, req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 public TSStatus alterLogicalView(final TAlterLogicalViewReq req) {
+1 final String queryId = req.getQueryId();
+1 final ByteBuffer byteBuffer = ByteBuffer.wrap(req.getViewBinary());
+1 final Map<PartialPath, ViewExpression> viewPathToSourceMap = new HashMap<>();
+1 final int size = byteBuffer.getInt();
+1 PartialPath path;
+1 ViewExpression viewExpression;
+1 for (int i = 0; i < size; i++) {
+1 path = (PartialPath) PathDeserializeUtil.deserialize(byteBuffer);
+1 viewExpression = ViewExpression.deserialize(byteBuffer);
+1 viewPathToSourceMap.put(path, viewExpression);
+1 }
+1
+1 AlterLogicalViewProcedure procedure = null;
+1 synchronized (this) {
+1 ProcedureType type;
+1 AlterLogicalViewProcedure alterLogicalViewProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.ALTER_LOGICAL_VIEW_PROCEDURE)) {
+1 continue;
+1 }
+1 alterLogicalViewProcedure = ((AlterLogicalViewProcedure) runningProcedure);
+1 if (queryId.equals(alterLogicalViewProcedure.getQueryId())) {
+1 procedure = alterLogicalViewProcedure;
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 procedure =
+1 new AlterLogicalViewProcedure(
+1 queryId,
+1 viewPathToSourceMap,
+1 req.isSetIsGeneratedByPipe() && req.isIsGeneratedByPipe());
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 public TSStatus setSchemaTemplate(
+1 String queryId, String templateName, String templateSetPath, boolean isGeneratedByPipe) {
+1 SetTemplateProcedure procedure = null;
+1 synchronized (this) {
+1 boolean hasOverlappedTask = false;
+1 ProcedureType type;
+1 SetTemplateProcedure setTemplateProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.SET_TEMPLATE_PROCEDURE)) {
+1 continue;
+1 }
+1 setTemplateProcedure = (SetTemplateProcedure) runningProcedure;
+1 if (queryId.equals(setTemplateProcedure.getQueryId())) {
+1 procedure = setTemplateProcedure;
+1 break;
+1 }
+1 if (templateSetPath.equals(setTemplateProcedure.getTemplateSetPath())) {
+1 hasOverlappedTask = true;
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 "Some other task is setting template on target path.");
+1 }
+1 procedure =
+1 new SetTemplateProcedure(queryId, templateName, templateSetPath, isGeneratedByPipe);
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 public TSStatus deactivateTemplate(
+1 String queryId, Map<PartialPath, List<Template>> templateSetInfo, boolean isGeneratedByPipe) {
+1 DeactivateTemplateProcedure procedure = null;
+1 synchronized (this) {
+1 boolean hasOverlappedTask = false;
+1 ProcedureType type;
+1 DeactivateTemplateProcedure deactivateTemplateProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.DEACTIVATE_TEMPLATE_PROCEDURE)) {
+1 continue;
+1 }
+1 deactivateTemplateProcedure = (DeactivateTemplateProcedure) runningProcedure;
+1 if (queryId.equals(deactivateTemplateProcedure.getQueryId())) {
+1 procedure = deactivateTemplateProcedure;
+1 break;
+1 }
+1 for (PartialPath pattern : templateSetInfo.keySet()) {
+1 for (PartialPath existingPattern :
+1 deactivateTemplateProcedure.getTemplateSetInfo().keySet()) {
+1 if (pattern.overlapWith(existingPattern)) {
+1 hasOverlappedTask = true;
+1 break;
+1 }
+1 }
+1 if (hasOverlappedTask) {
+1 break;
+1 }
+1 }
+1 if (hasOverlappedTask) {
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 "Some other task is deactivating some target template from target path.");
+1 }
+1 procedure = new DeactivateTemplateProcedure(queryId, templateSetInfo, isGeneratedByPipe);
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 public TSStatus unsetSchemaTemplate(
+1 String queryId, Template template, PartialPath path, boolean isGeneratedByPipe) {
+1 UnsetTemplateProcedure procedure = null;
+1 synchronized (this) {
+1 boolean hasOverlappedTask = false;
+1 ProcedureType type;
+1 UnsetTemplateProcedure unsetTemplateProcedure;
+1 for (Procedure<?> runningProcedure : executor.getProcedures().values()) {
+1 type = ProcedureFactory.getProcedureType(runningProcedure);
+1 if (type == null || !type.equals(ProcedureType.UNSET_TEMPLATE_PROCEDURE)) {
+1 continue;
+1 }
+1 unsetTemplateProcedure = (UnsetTemplateProcedure) runningProcedure;
+1 if (queryId.equals(unsetTemplateProcedure.getQueryId())) {
+1 procedure = unsetTemplateProcedure;
+1 break;
+1 }
+1 if (template.getId() == unsetTemplateProcedure.getTemplateId()
+1 && path.equals(unsetTemplateProcedure.getPath())) {
+1 hasOverlappedTask = true;
+1 break;
+1 }
+1 }
+1
+1 if (procedure == null) {
+1 if (hasOverlappedTask) {
+1 return RpcUtils.getStatus(
+1 TSStatusCode.OVERLAP_WITH_EXISTING_TASK,
+1 "Some other task is unsetting target template from target path "
+1 + path.getFullPath());
+1 }
+1 procedure = new UnsetTemplateProcedure(queryId, template, path, isGeneratedByPipe);
+1 this.executor.submitProcedure(procedure);
+1 }
+1 }
+1 return waitingProcedureFinished(procedure);
+1 }
+1
+1 /**
+1 * Generate an {@link AddConfigNodeProcedure}, and serially execute all the {@link
+1 * AddConfigNodeProcedure}s.
+1 */
+1 public void addConfigNode(TConfigNodeRegisterReq req) {
+1 final AddConfigNodeProcedure addConfigNodeProcedure =
+1 new AddConfigNodeProcedure(req.getConfigNodeLocation(), req.getVersionInfo());
+1 this.executor.submitProcedure(addConfigNodeProcedure);
+1 }
+1
+1 /**
+1 * Generate a {@link RemoveConfigNodeProcedure}, and serially execute all the {@link
+1 * RemoveConfigNodeProcedure}s.
+1 */
+1 public void removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan) {
+1 final RemoveConfigNodeProcedure removeConfigNodeProcedure =
+1 new RemoveConfigNodeProcedure(removeConfigNodePlan.getConfigNodeLocation());
+1 this.executor.submitProcedure(removeConfigNodeProcedure);
+1 LOGGER.info("Submit RemoveConfigNodeProcedure successfully: {}", removeConfigNodePlan);
+1 }
+1
+1 /**
+1 * Generate {@link RemoveDataNodesProcedure}s, and serially execute all the {@link
+1 * RemoveDataNodesProcedure}s.
+1 */
+1 public boolean removeDataNode(RemoveDataNodePlan removeDataNodePlan) {
+1 Map<Integer, NodeStatus> nodeStatusMap = new HashMap<>();
+1 removeDataNodePlan
+1 .getDataNodeLocations()
+1 .forEach(
+1 datanode ->
+1 nodeStatusMap.put(
+1 datanode.getDataNodeId(),
+1 configManager.getLoadManager().getNodeStatus(datanode.getDataNodeId())));
+1 this.executor.submitProcedure(
+1 new RemoveDataNodesProcedure(removeDataNodePlan.getDataNodeLocations(), nodeStatusMap));
+1 LOGGER.info(
+1 "Submit RemoveDataNodesProcedure successfully, {}",
+1 removeDataNodePlan.getDataNodeLocations());
+1 return true;
+1 }
+1
+1 public boolean removeAINode(RemoveAINodePlan removeAINodePlan) {
+1 this.executor.submitProcedure(new RemoveAINodeProcedure(removeAINodePlan.getAINodeLocation()));
+1 LOGGER.info(
+1 "Submit RemoveAINodeProcedure successfully, {}", removeAINodePlan.getAINodeLocation());
+1 return true;
+1 }
+1
+1 public TSStatus checkRemoveDataNodes(List<TDataNodeLocation> dataNodeLocations) {
+1 // 1. Only one RemoveDataNodesProcedure is allowed in the cluster
+1 Optional<Procedure<ConfigNodeProcedureEnv>> anotherRemoveProcedure =
+1 getExecutor().getProcedures().values().stream()
+1 .filter(
+1 procedure -> {
+1 if (procedure instanceof RemoveDataNodesProcedure) {
+1 return !procedure.isFinished();
+1 }
+1 return false;
+1 })
+1 .findAny();
+1
+1 String failMessage = null;
+1 if (anotherRemoveProcedure.isPresent()) {
+1 List<TDataNodeLocation> anotherRemoveDataNodes =
+1 ((RemoveDataNodesProcedure) anotherRemoveProcedure.get()).getRemovedDataNodes();
+1 failMessage =
+1 String.format(
+1 "Submit RemoveDataNodesProcedure failed, "
+1 + "because another RemoveDataNodesProcedure %s is already in processing. "
+1 + "IoTDB is able to have at most 1 RemoveDataNodesProcedure at the same time. "
+1 + "For further information, please search [pid%d] in log. ",
+1 anotherRemoveDataNodes, anotherRemoveProcedure.get().getProcId());
+1 }
+1
+1 // 2. Check if the RemoveDataNodesProcedure conflicts with the RegionMigrateProcedure
+1 Set<TConsensusGroupId> removedDataNodesRegionSet =
+1 getEnv().getRemoveDataNodeHandler().getRemovedDataNodesRegionSet(dataNodeLocations);
+1 Optional<Procedure<ConfigNodeProcedureEnv>> conflictRegionMigrateProcedure =
+1 getExecutor().getProcedures().values().stream()
+1 .filter(
+1 procedure -> {
+1 if (procedure instanceof RegionMigrateProcedure) {
+1 RegionMigrateProcedure regionMigrateProcedure =
+1 (RegionMigrateProcedure) procedure;
+1 if (regionMigrateProcedure.isFinished()) {
+1 return false;
+1 }
+1 return removedDataNodesRegionSet.contains(regionMigrateProcedure.getRegionId())
+1 || dataNodeLocations.contains(regionMigrateProcedure.getDestDataNode());
+1 }
+1 return false;
+1 })
+1 .findAny();
+1 if (conflictRegionMigrateProcedure.isPresent()) {
+1 failMessage =
+1 String.format(
+1 "Submit RemoveDataNodesProcedure failed, "
+1 + "because another RegionMigrateProcedure %s is already in processing which conflicts with this RemoveDataNodesProcedure. "
+1 + "The RegionMigrateProcedure is migrating the region %s to the DataNode %s. "
+1 + "For further information, please search [pid%d] in log. ",
+1 conflictRegionMigrateProcedure.get().getProcId(),
+1 ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getRegionId(),
+1 ((RegionMigrateProcedure) conflictRegionMigrateProcedure.get()).getDestDataNode(),
+1 conflictRegionMigrateProcedure.get().getProcId());
+1 }
+1 // 3. Check if the RegionMigrateProcedure generated by RemoveDataNodesProcedure conflicts with
+1 // each other
+1 List<RegionMigrationPlan> regionMigrationPlans =
+1 getEnv().getRemoveDataNodeHandler().getRegionMigrationPlans(dataNodeLocations);
+1 removedDataNodesRegionSet.clear();
+1 for (RegionMigrationPlan regionMigrationPlan : regionMigrationPlans) {
+1 if (removedDataNodesRegionSet.contains(regionMigrationPlan.getRegionId())) {
+1 failMessage =
+1 String.format(
+1 "Submit RemoveDataNodesProcedure failed, "
+1 + "because the RegionMigrateProcedure generated by this RemoveDataNodesProcedure conflicts with each other. "
+1 + "Only one replica of the same consensus group is allowed to be migrated at the same time."
+1 + "The conflict region id is %s . ",
+1 regionMigrationPlan.getRegionId());
+1 break;
+1 }
+1 removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId());
+1 }
+1
+1 // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are
+1 // not the remove DataNodes
+1
+1 for (TDataNodeLocation removeDataNode : dataNodeLocations) {
+1 Set<TDataNodeLocation> relatedDataNodes =
+1 getEnv().getRemoveDataNodeHandler().getRelatedDataNodeLocations(removeDataNode);
+1 relatedDataNodes.remove(removeDataNode);
+1
+1 for (TDataNodeLocation relatedDataNode : relatedDataNodes) {
+1 NodeStatus nodeStatus =
+1 getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId());
+1 if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) {
+1 failMessage =
+1 String.format(
+1 "Submit RemoveDataNodesProcedure failed, "
+1 + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, "
+1 + "the remove operation cannot be performed for security reasons. "
+1 + "Please check the status of the node %s and ensure it is running.",
+1 relatedDataNode.getDataNodeId());
+1 }
+1 }
+1 }
+1
+1 if (failMessage != null) {
+1 LOGGER.warn(failMessage);
+1 TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_DATANODE_ERROR.getStatusCode());
+1 failStatus.setMessage(failMessage);
+1 return failStatus;
+1 }
+1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 // region region operation related check
+1
+1 /**
+1 * Checks whether region migration is allowed.
+1 *
+1 * @param migrateRegionReq the migration request details
+1 * @param regionGroupId the ID of the consensus group for the region
+1 * @param originalDataNode the original DataNode location from which the region is being migrated
+1 * @param destDataNode the destination DataNode location to which the region is being migrated
+1 * @param coordinatorForAddPeer the DataNode location acting as the coordinator for adding a peer
+1 * @return the status of the migration request (TSStatus)
+1 */
+1 private TSStatus checkMigrateRegion(
+1 TMigrateRegionReq migrateRegionReq,
+1 TConsensusGroupId regionGroupId,
+1 TDataNodeLocation originalDataNode,
+1 TDataNodeLocation destDataNode,
+1 TDataNodeLocation coordinatorForAddPeer) {
+1 String failMessage;
+1 if ((failMessage =
+1 regionOperationCommonCheck(
+1 regionGroupId,
+1 destDataNode,
+1 Arrays.asList(
+1 new Pair<>("Original DataNode", originalDataNode),
+1 new Pair<>("Destination DataNode", destDataNode),
+1 new Pair<>("Coordinator for add peer", coordinatorForAddPeer)),
+1 migrateRegionReq.getModel()))
+1 != null) {
+1 // do nothing
+1 } else if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSets(originalDataNode.getDataNodeId())
+1 .stream()
+1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
+1 failMessage =
+1 String.format(
+1 "Submit RegionMigrateProcedure failed, because the original DataNode %s doesn't contain Region %s",
+1 migrateRegionReq.getFromId(), migrateRegionReq.getRegionId());
+1 } else if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSets(destDataNode.getDataNodeId())
+1 .stream()
+1 .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionGroupId))) {
+1 failMessage =
+1 String.format(
+1 "Submit RegionMigrateProcedure failed, because the target DataNode %s already contains Region %s",
+1 migrateRegionReq.getToId(), migrateRegionReq.getRegionId());
+1 }
+1
+1 if (failMessage != null) {
+1 LOGGER.warn(failMessage);
+1 TSStatus failStatus = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+1 failStatus.setMessage(failMessage);
+1 return failStatus;
+1 }
+1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 private TSStatus checkReconstructRegion(
+1 TReconstructRegionReq req,
+1 TConsensusGroupId regionId,
+1 TDataNodeLocation targetDataNode,
+1 TDataNodeLocation coordinator) {
+1 String failMessage =
+1 regionOperationCommonCheck(
+1 regionId,
+1 targetDataNode,
+1 Arrays.asList(
+1 new Pair<>("Target DataNode", targetDataNode),
+1 new Pair<>("Coordinator", coordinator)),
+1 req.getModel());
+1
+1 ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
+1 if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSetsMap(regionId.getType())
+1 .get(regionId)
+1 .getDataNodeLocationsSize()
+1 == 1) {
+1 failMessage = String.format("%s only has 1 replica, it cannot be reconstructed", regionId);
+1 } else if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSets(targetDataNode.getDataNodeId())
+1 .stream()
+1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+1 failMessage =
+1 String.format(
+1 "Submit ReconstructRegionProcedure failed, because the target DataNode %s doesn't contain Region %s",
+1 req.getDataNodeId(), regionId);
+1 }
+1
+1 if (failMessage != null) {
+1 LOGGER.warn(failMessage);
+1 TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
+1 failStatus.setMessage(failMessage);
+1 return failStatus;
+1 }
+1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 private TSStatus checkExtendRegion(
+1 TExtendRegionReq req,
+1 TConsensusGroupId regionId,
+1 TDataNodeLocation targetDataNode,
+1 TDataNodeLocation coordinator) {
+1 String failMessage =
+1 regionOperationCommonCheck(
+1 regionId,
+1 targetDataNode,
+1 Arrays.asList(
+1 new Pair<>("Target DataNode", targetDataNode),
+1 new Pair<>("Coordinator", coordinator)),
+1 req.getModel());
+1 if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSets(targetDataNode.getDataNodeId())
+1 .stream()
+1 .anyMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+1 failMessage =
+1 String.format(
+1 "Target DataNode %s already contains region %s",
+1 targetDataNode.getDataNodeId(), regionId);
+1 }
+1
+1 if (failMessage != null) {
+1 LOGGER.warn(failMessage);
+1 TSStatus failStatus = new TSStatus(TSStatusCode.RECONSTRUCT_REGION_ERROR.getStatusCode());
+1 failStatus.setMessage(failMessage);
+1 return failStatus;
+1 }
+1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 private TSStatus checkRemoveRegion(
+1 TRemoveRegionReq req,
+1 TConsensusGroupId regionId,
+1 @Nullable TDataNodeLocation targetDataNode,
+1 TDataNodeLocation coordinator) {
+1 String failMessage =
+1 regionOperationCommonCheck(
+1 regionId,
+1 targetDataNode,
+1 Arrays.asList(new Pair<>("Coordinator", coordinator)),
+1 req.getModel());
+1
+1 if (configManager
+1 .getPartitionManager()
+1 .getAllReplicaSetsMap(regionId.getType())
+1 .get(regionId)
+1 .getDataNodeLocationsSize()
+1 == 1) {
+1 failMessage = String.format("%s only has 1 replica, it cannot be removed", regionId);
+1 } else if (targetDataNode != null
+1 && configManager
+1 .getPartitionManager()
+1 .getAllReplicaSets(targetDataNode.getDataNodeId())
+1 .stream()
+1 .noneMatch(replicaSet -> replicaSet.getRegionId().equals(regionId))) {
+1 failMessage =
+1 String.format(
+1 "Target DataNode %s doesn't contain Region %s", req.getDataNodeId(), regionId);
+1 }
+1
+1 if (failMessage != null) {
+1 LOGGER.warn(failMessage);
+1 TSStatus failStatus = new TSStatus(TSStatusCode.REMOVE_REGION_PEER_ERROR.getStatusCode());
+1 failStatus.setMessage(failMessage);
+1 return failStatus;
+1 }
+1 return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 /**
+1 * The common checks of all region operations, include migration, reconstruction, extension,
+1 * removing
+1 *
+1 * @param regionId region group id, also called consensus group id
+1 * @param targetDataNode DataNode should in Running status
+1 * @param relatedDataNodes Pair<Identity, Node Location>
+1 * @return The reason if check failed, or null if check pass
+1 */
+1 private String regionOperationCommonCheck(
+1 TConsensusGroupId regionId,
+1 TDataNodeLocation targetDataNode,
+1 List<Pair<String, TDataNodeLocation>> relatedDataNodes,
+1 Model model) {
+1 String failMessage;
+1 ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
+1
+1 if (TConsensusGroupType.DataRegion == regionId.getType()
+1 && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getDataRegionConsensusProtocolClass())) {
+1 failMessage = "SimpleConsensus not supports region operation.";
+1 } else if (TConsensusGroupType.SchemaRegion == regionId.getType()
+1 && ConsensusFactory.SIMPLE_CONSENSUS.equals(conf.getSchemaRegionConsensusProtocolClass())) {
+1 failMessage = "SimpleConsensus not supports region operation.";
+1 } else if ((failMessage = checkRegionOperationDuplication(regionId)) != null) {
+1 // need to do nothing more
+1 } else if (relatedDataNodes.stream().anyMatch(pair -> pair.getRight() == null)) {
+1 Pair<String, TDataNodeLocation> nullPair =
+1 relatedDataNodes.stream().filter(pair -> pair.getRight() == null).findAny().get();
+1 failMessage = String.format("Cannot find %s", nullPair.getLeft());
+1 } else if (targetDataNode != null
+1 && !configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running).stream()
+1 .map(TDataNodeConfiguration::getLocation)
+1 .map(TDataNodeLocation::getDataNodeId)
+1 .collect(Collectors.toSet())
+1 .contains(targetDataNode.getDataNodeId())) {
+1 // Here we only check Running DataNode to implement migration, because removing nodes may not
+1 // exist when add peer is performing
+1 failMessage =
+1 String.format(
+1 "Target DataNode %s is not in Running status.", targetDataNode.getDataNodeId());
+1 } else if ((failMessage = checkRegionOperationWithRemoveDataNode(regionId, targetDataNode))
+1 != null) {
+1 // need to do nothing more
+1 } else if ((failMessage = checkRegionOperationModelCorrectness(regionId, model)) != null) {
+1 // need to do nothing more
+1 }
+1
+1 return failMessage;
+1 }
+1
+1 private String checkRegionOperationWithRemoveDataNode(
+1 TConsensusGroupId regionId, TDataNodeLocation targetDataNode) {
+1 Optional<Procedure<ConfigNodeProcedureEnv>> conflictRemoveDataNodesProcedure =
+1 getExecutor().getProcedures().values().stream()
+1 .filter(
+1 procedure -> {
+1 if (procedure instanceof RemoveDataNodesProcedure) {
+1 return !procedure.isFinished();
+1 }
+1 return false;
+1 })
+1 .findAny();
+1
+1 if (conflictRemoveDataNodesProcedure.isPresent()) {
+1 RemoveDataNodeHandler removeDataNodeHandler = env.getRemoveDataNodeHandler();
+1 List<TDataNodeLocation> removedDataNodes =
+1 ((RemoveDataNodesProcedure) conflictRemoveDataNodesProcedure.get()).getRemovedDataNodes();
+1 Set<TConsensusGroupId> removedDataNodesRegionSet =
+1 removeDataNodeHandler.getRemovedDataNodesRegionSet(removedDataNodes);
+1 if (removedDataNodesRegionSet.contains(regionId)) {
+1 return String.format(
+1 "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
+1 + "The RemoveDataNodesProcedure is removing the DataNodes %s which contains the region %s. "
+1 + "For further information, please search [pid%d] in log. ",
+1 conflictRemoveDataNodesProcedure.get().getProcId(),
+1 removedDataNodes,
+1 regionId,
+1 conflictRemoveDataNodesProcedure.get().getProcId());
+1 } else if (removedDataNodes.contains(targetDataNode)) {
+1 return String.format(
+1 "Another RemoveDataNodesProcedure %s is already in processing which conflicts with this procedure. "
+1 + "The RemoveDataNodesProcedure is removing the target DataNode %s. "
+1 + "For further information, please search [pid%d] in log. ",
+1 conflictRemoveDataNodesProcedure.get().getProcId(),
+1 targetDataNode,
+1 conflictRemoveDataNodesProcedure.get().getProcId());
+1 }
+1 }
+1 return null;
+1 }
+1
+1 private String checkRegionOperationDuplication(TConsensusGroupId regionId) {
+1 List<? extends RegionOperationProcedure<?>> otherRegionMemberChangeProcedures =
+1 getRegionOperationProcedures()
+1 .filter(
+1 regionMemberChangeProcedure ->
+1 regionId.equals(regionMemberChangeProcedure.getRegionId()))
+1 .collect(Collectors.toList());
+1 if (!otherRegionMemberChangeProcedures.isEmpty()) {
+1 return String.format(
+1 "%s has some other region operation procedures in progress, their procedure id is: %s",
+1 regionId, otherRegionMemberChangeProcedures);
+1 }
+1 return null;
+1 }
+1
+1 public List<TConsensusGroupId> getRegionOperationConsensusIds() {
+1 return getRegionOperationProcedures()
+1 .map(RegionOperationProcedure::getRegionId)
+1 .distinct()
+1 .collect(Collectors.toList());
+1 }
+1
+1 private Stream<RegionOperationProcedure<?>> getRegionOperationProcedures() {
+1 return getExecutor().getProcedures().values().stream()
+1 .filter(procedure -> !procedure.isFinished())
+1 .filter(procedure -> procedure instanceof RegionOperationProcedure)
+1 .map(procedure -> (RegionOperationProcedure<?>) procedure);
+1 }
+1
+1 private String checkRegionOperationModelCorrectness(TConsensusGroupId regionId, Model model) {
+1 String databaseName = configManager.getPartitionManager().getRegionDatabase(regionId);
boolean isTreeModelDatabase = databaseName.startsWith(SqlConstant.TREE_MODEL_DATABASE_PREFIX);
if (Model.TREE == model && isTreeModelDatabase
|| Model.TABLE == model && !isTreeModelDatabase) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
index ba6712a983d..b61cdf46535 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureContainer.java
@@ -1,36 +1,37 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-public class CompletedProcedureContainer<Env> {
- private final Procedure<Env> procedure;
-
- public CompletedProcedureContainer(Procedure<Env> procedure) {
- this.procedure = procedure;
- }
-
- public Procedure<Env> getProcedure() {
- return procedure;
- }
-
- public boolean isExpired(long now, long evictTtl) {
- return (now - procedure.getLastUpdate()) >= evictTtl;
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1public class CompletedProcedureContainer<Env> {
+1 private final Procedure<Env> procedure;
+1
+1 public CompletedProcedureContainer(Procedure<Env> procedure) {
+1 this.procedure = procedure;
+1 }
+1
+1 public Procedure<Env> getProcedure() {
+1 return procedure;
+1 }
+1
+1 public boolean isExpired(long now, long evictTtl) {
+1 return (now - procedure.getLastUpdate()) >= evictTtl;
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
index cea15a1bfb6..aea91dbdb22 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/CompletedProcedureRecycler.java
@@ -1,84 +1,85 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-/** Internal cleaner that removes the completed procedure results after a TTL. */
-public class CompletedProcedureRecycler<Env> extends InternalProcedure<Env> {
- private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureRecycler.class);
- private static final int DEFAULT_BATCH_SIZE = 8;
- private final long evictTTL;
- private final Map<Long, CompletedProcedureContainer<Env>> completed;
- private final IProcedureStore<Env> store;
-
- public CompletedProcedureRecycler(
- IProcedureStore<Env> store,
- Map<Long, CompletedProcedureContainer<Env>> completedMap,
- long cleanTimeInterval,
- long evictTTL) {
- super(TimeUnit.SECONDS.toMillis(cleanTimeInterval));
- this.completed = completedMap;
- this.store = store;
- this.evictTTL = evictTTL;
- }
-
- @Override
- protected void periodicExecute(final Env env) {
- if (completed.isEmpty()) {
- if (LOG.isTraceEnabled()) {
- LOG.trace("No completed procedures to cleanup.");
- }
- return;
- }
-
- final long[] batchIds = new long[DEFAULT_BATCH_SIZE];
- int batchCount = 0;
-
- final long now = System.currentTimeMillis();
- final Iterator<Map.Entry<Long, CompletedProcedureContainer<Env>>> it =
- completed.entrySet().iterator();
- while (it.hasNext() && store.isRunning()) {
- final Map.Entry<Long, CompletedProcedureContainer<Env>> entry = it.next();
- final CompletedProcedureContainer<Env> retainer = entry.getValue();
- final Procedure<?> proc = retainer.getProcedure();
- if (retainer.isExpired(now, evictTTL)) {
- // Failed procedures aren't persisted in WAL.
- batchIds[batchCount++] = entry.getKey();
- if (batchCount == batchIds.length) {
- store.delete(batchIds, 0, batchCount);
- batchCount = 0;
- }
- it.remove();
- LOG.trace("Evict completed {}", proc);
- }
- }
- if (batchCount > 0) {
- store.delete(batchIds, 0, batchCount);
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.util.Iterator;
+1import java.util.Map;
+1import java.util.concurrent.TimeUnit;
+1
+1/** Internal cleaner that removes the completed procedure results after a TTL. */
+1public class CompletedProcedureRecycler<Env> extends InternalProcedure<Env> {
+1 private static final Logger LOG = LoggerFactory.getLogger(CompletedProcedureRecycler.class);
+1 private static final int DEFAULT_BATCH_SIZE = 8;
+1 private final long evictTTL;
+1 private final Map<Long, CompletedProcedureContainer<Env>> completed;
+1 private final IProcedureStore<Env> store;
+1
+1 public CompletedProcedureRecycler(
+1 IProcedureStore<Env> store,
+1 Map<Long, CompletedProcedureContainer<Env>> completedMap,
+1 long cleanTimeInterval,
+1 long evictTTL) {
+1 super(TimeUnit.SECONDS.toMillis(cleanTimeInterval));
+1 this.completed = completedMap;
+1 this.store = store;
+1 this.evictTTL = evictTTL;
+1 }
+1
+1 @Override
+1 protected void periodicExecute(final Env env) {
+1 if (completed.isEmpty()) {
+1 if (LOG.isTraceEnabled()) {
+1 LOG.trace("No completed procedures to cleanup.");
+1 }
+1 return;
+1 }
+1
+1 final long[] batchIds = new long[DEFAULT_BATCH_SIZE];
+1 int batchCount = 0;
+1
+1 final long now = System.currentTimeMillis();
+1 final Iterator<Map.Entry<Long, CompletedProcedureContainer<Env>>> it =
+1 completed.entrySet().iterator();
+1 while (it.hasNext() && store.isRunning()) {
+1 final Map.Entry<Long, CompletedProcedureContainer<Env>> entry = it.next();
+1 final CompletedProcedureContainer<Env> retainer = entry.getValue();
+1 final Procedure<?> proc = retainer.getProcedure();
+1 if (retainer.isExpired(now, evictTTL)) {
+1 // Failed procedures aren't persisted in WAL.
+1 batchIds[batchCount++] = entry.getKey();
+1 if (batchCount == batchIds.length) {
+1 store.delete(batchIds, 0, batchCount);
+1 batchCount = 0;
+1 }
+1 it.remove();
+1 LOG.trace("Evict completed {}", proc);
+1 }
+1 }
+1 if (batchCount > 0) {
+1 store.delete(batchIds, 0, batchCount);
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
index f9ff68bde7e..4a70aed89d0 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/InternalProcedure.java
@@ -1,49 +1,50 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-/**
- * Internal Procedure, do some periodic job for framework.
- *
- * @param <Env>
- */
-public abstract class InternalProcedure<Env> extends Procedure<Env> {
- protected InternalProcedure(long toMillis) {
- setTimeout(toMillis);
- }
-
- protected abstract void periodicExecute(final Env env);
-
- @Override
- protected Procedure<Env>[] execute(Env env) throws InterruptedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- protected void rollback(Env env) throws IOException, InterruptedException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void deserialize(ByteBuffer byteBuffer) {}
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import java.io.IOException;
+1import java.nio.ByteBuffer;
+1
+1/**
+1 * Internal Procedure, do some periodic job for framework.
+1 *
+1 * @param <Env>
+1 */
+1public abstract class InternalProcedure<Env> extends Procedure<Env> {
+1 protected InternalProcedure(long toMillis) {
+1 setTimeout(toMillis);
+1 }
+1
+1 protected abstract void periodicExecute(final Env env);
+1
+1 @Override
+1 protected Procedure<Env>[] execute(Env env) throws InterruptedException {
+1 throw new UnsupportedOperationException();
+1 }
+1
+1 @Override
+1 protected void rollback(Env env) throws IOException, InterruptedException {
+1 throw new UnsupportedOperationException();
+1 }
+1
+1 @Override
+1 public void deserialize(ByteBuffer byteBuffer) {}
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
index 2ab15283207..4d2d8a23256 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/PartitionTableAutoCleaner.java
@@ -1,112 +1,113 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
-import org.apache.iotdb.commons.conf.CommonConfig;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
-import org.apache.iotdb.commons.utils.PathUtils;
-import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
-import org.apache.iotdb.confignode.manager.ConfigManager;
-import org.apache.iotdb.consensus.exception.ConsensusException;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-import static org.apache.iotdb.confignode.manager.partition.PartitionManager.CONSENSUS_WRITE_ERROR;
-
-/** A cleaner that automatically deletes the expired mapping within the partition table. */
-public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
-
- private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
-
- private static final String timestampPrecision =
- CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
-
- private final ConfigManager configManager;
-
- public PartitionTableAutoCleaner(ConfigManager configManager) {
- super(COMMON_CONFIG.getTTLCheckInterval());
- this.configManager = configManager;
- LOGGER.info(
- "[PartitionTableCleaner] The PartitionTableAutoCleaner is started with cycle={}ms",
- COMMON_CONFIG.getTTLCheckInterval());
- }
-
- @Override
- protected void periodicExecute(Env env) {
- List<String> databases = configManager.getClusterSchemaManager().getDatabaseNames(null);
- Map<String, Long> databaseTTLMap = new TreeMap<>();
- for (String database : databases) {
- long databaseTTL =
- PathUtils.isTableModelDatabase(database)
- ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database)
- : configManager.getTTLManager().getDatabaseMaxTTL(database);
- databaseTTLMap.put(database, databaseTTL);
- }
- LOGGER.info(
- "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}",
- databaseTTLMap);
- for (String database : databases) {
- long databaseTTL = databaseTTLMap.get(database);
- if (!configManager.getPartitionManager().isDatabaseExist(database)
- || databaseTTL < 0
- || databaseTTL == Long.MAX_VALUE) {
- // Remove the entry if the database or the TTL does not exist
- databaseTTLMap.remove(database);
- }
- }
- if (!databaseTTLMap.isEmpty()) {
- LOGGER.info(
- "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
- databaseTTLMap);
- // Only clean the partition table when necessary
- TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
- try {
- configManager
- .getConsensusManager()
- .write(new AutoCleanPartitionTablePlan(databaseTTLMap, currentTimePartitionSlot));
- } catch (ConsensusException e) {
- LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
- }
- }
- }
-
- /**
- * @return The time partition slot corresponding to current timestamp. Note that we do not shift
- * the start time to the correct starting point, since this interface only constructs a time
- * reference position for the partition table cleaner.
- */
- private static TTimePartitionSlot getCurrentTimePartitionSlot() {
- if ("ms".equals(timestampPrecision)) {
- return new TTimePartitionSlot(System.currentTimeMillis());
- } else if ("us".equals(timestampPrecision)) {
- return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
- } else {
- return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
+1import org.apache.iotdb.commons.conf.CommonConfig;
+1import org.apache.iotdb.commons.conf.CommonDescriptor;
+1import org.apache.iotdb.commons.utils.PathUtils;
+1import org.apache.iotdb.confignode.consensus.request.write.partition.AutoCleanPartitionTablePlan;
+1import org.apache.iotdb.confignode.manager.ConfigManager;
+1import org.apache.iotdb.consensus.exception.ConsensusException;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.util.List;
+1import java.util.Map;
+1import java.util.TreeMap;
+1
+1import static org.apache.iotdb.confignode.manager.partition.PartitionManager.CONSENSUS_WRITE_ERROR;
+1
+1/** A cleaner that automatically deletes the expired mapping within the partition table. */
+1public class PartitionTableAutoCleaner<Env> extends InternalProcedure<Env> {
+1
+1 private static final Logger LOGGER = LoggerFactory.getLogger(PartitionTableAutoCleaner.class);
+1
+1 private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
+1
+1 private static final String timestampPrecision =
+1 CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+1
+1 private final ConfigManager configManager;
+1
+1 public PartitionTableAutoCleaner(ConfigManager configManager) {
+1 super(COMMON_CONFIG.getTTLCheckInterval());
+1 this.configManager = configManager;
+1 LOGGER.info(
+1 "[PartitionTableCleaner] The PartitionTableAutoCleaner is started with cycle={}ms",
+1 COMMON_CONFIG.getTTLCheckInterval());
+1 }
+1
+1 @Override
+1 protected void periodicExecute(Env env) {
+1 List<String> databases = configManager.getClusterSchemaManager().getDatabaseNames(null);
+1 Map<String, Long> databaseTTLMap = new TreeMap<>();
+1 for (String database : databases) {
+1 long databaseTTL =
+1 PathUtils.isTableModelDatabase(database)
+1 ? configManager.getClusterSchemaManager().getDatabaseMaxTTL(database)
+1 : configManager.getTTLManager().getDatabaseMaxTTL(database);
+1 databaseTTLMap.put(database, databaseTTL);
+1 }
+1 LOGGER.info(
+1 "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner, databaseTTL: {}",
+1 databaseTTLMap);
+1 for (String database : databases) {
+1 long databaseTTL = databaseTTLMap.get(database);
+1 if (!configManager.getPartitionManager().isDatabaseExist(database)
+1 || databaseTTL < 0
+1 || databaseTTL == Long.MAX_VALUE) {
+1 // Remove the entry if the database or the TTL does not exist
+1 databaseTTLMap.remove(database);
+1 }
+1 }
+1 if (!databaseTTLMap.isEmpty()) {
+1 LOGGER.info(
+1 "[PartitionTableCleaner] Periodically activate PartitionTableAutoCleaner for: {}",
+1 databaseTTLMap);
+1 // Only clean the partition table when necessary
+1 TTimePartitionSlot currentTimePartitionSlot = getCurrentTimePartitionSlot();
+1 try {
+1 configManager
+1 .getConsensusManager()
+1 .write(new AutoCleanPartitionTablePlan(databaseTTLMap, currentTimePartitionSlot));
+1 } catch (ConsensusException e) {
+1 LOGGER.warn(CONSENSUS_WRITE_ERROR, e);
+1 }
+1 }
+1 }
+1
+1 /**
+1 * @return The time partition slot corresponding to current timestamp. Note that we do not shift
+1 * the start time to the correct starting point, since this interface only constructs a time
+1 * reference position for the partition table cleaner.
+1 */
+1 private static TTimePartitionSlot getCurrentTimePartitionSlot() {
+1 if ("ms".equals(timestampPrecision)) {
+1 return new TTimePartitionSlot(System.currentTimeMillis());
+1 } else if ("us".equals(timestampPrecision)) {
+1 return new TTimePartitionSlot(System.currentTimeMillis() * 1000);
+1 } else {
+1 return new TTimePartitionSlot(System.currentTimeMillis() * 1000_000);
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
index 91af03d3971..7eb9eaea987 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/Procedure.java
@@ -1,823 +1,824 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
-import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * Abstract class of all procedures.
- *
- * @param <Env>
- */
-public abstract class Procedure<Env> implements Comparable<Procedure<Env>> {
- private static final Logger LOG = LoggerFactory.getLogger(Procedure.class);
- public static final long NO_PROC_ID = -1;
- public static final long NO_TIMEOUT = -1;
-
- private long parentProcId = NO_PROC_ID;
- private long rootProcId = NO_PROC_ID;
- private long procId = NO_PROC_ID;
- private long submittedTime;
-
- private ProcedureState state = ProcedureState.INITIALIZING;
- private int childrenLatch = 0;
- private ProcedureException exception;
-
- private volatile long timeout = NO_TIMEOUT;
- private volatile long lastUpdate;
-
- private final AtomicReference<byte[]> result = new AtomicReference<>();
- private volatile boolean locked = false;
- private boolean lockedWhenLoading = false;
-
- private int[] stackIndexes = null;
-
- public final boolean hasLock() {
- return locked;
- }
-
- // User level code, override it if necessary
-
- /**
- * The main code of the procedure. It must be idempotent since execute() may be called multiple
- * times in case of machine failure in the middle of the execution.
- *
- * @param env the environment passed to the ProcedureExecutor
- * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the
- * procedure is done.
- * @throws InterruptedException the procedure will be added back to the queue and retried later.
- */
- protected abstract Procedure<Env>[] execute(Env env) throws InterruptedException;
-
- /**
- * The code to undo what was done by the execute() code. It is called when the procedure or one of
- * the sub-procedures failed or an abort was requested. It should cleanup all the resources
- * created by the execute() call. The implementation must be idempotent since rollback() may be
- * called multiple time in case of machine failure in the middle of the execution.
- *
- * @param env the environment passed to the ProcedureExecutor
- * @throws IOException temporary failure, the rollback will retry later
- * @throws InterruptedException the procedure will be added back to the queue and retried later
- */
- protected abstract void rollback(Env env)
- throws IOException, InterruptedException, ProcedureException;
-
- public void serialize(DataOutputStream stream) throws IOException {
- // procid
- stream.writeLong(this.procId);
- // state
- stream.writeInt(this.state.ordinal());
- // submit time
- stream.writeLong(this.submittedTime);
- // last updated
- stream.writeLong(this.lastUpdate);
- // parent id
- stream.writeLong(this.parentProcId);
- // time out
- stream.writeLong(this.timeout);
- // stack indexes
- if (stackIndexes != null) {
- stream.writeInt(stackIndexes.length);
- for (int index : stackIndexes) {
- stream.writeInt(index);
- }
- } else {
- stream.writeInt(-1);
- }
-
- // exceptions
- if (hasException()) {
- // symbol of exception
- stream.write((byte) 1);
- // exception's name
- String exceptionClassName = exception.getClass().getName();
- byte[] exceptionClassNameBytes = exceptionClassName.getBytes(StandardCharsets.UTF_8);
- stream.writeInt(exceptionClassNameBytes.length);
- stream.write(exceptionClassNameBytes);
- // exception's message
- String message = this.exception.getMessage();
- if (message != null) {
- byte[] messageBytes = message.getBytes(StandardCharsets.UTF_8);
- stream.writeInt(messageBytes.length);
- stream.write(messageBytes);
- } else {
- stream.writeInt(-1);
- }
- } else {
- // symbol of no exception
- stream.write((byte) 0);
- }
-
- // result
- if (result.get() != null) {
- stream.writeInt(result.get().length);
- stream.write(result.get());
- } else {
- stream.writeInt(-1);
- }
-
- // Has lock
- stream.write(this.hasLock() ? (byte) 1 : (byte) 0);
- }
-
- public void deserialize(ByteBuffer byteBuffer) {
- // procid
- this.setProcId(byteBuffer.getLong());
- // state
- this.setState(ProcedureState.values()[byteBuffer.getInt()]);
- // submit time
- this.setSubmittedTime(byteBuffer.getLong());
- // last updated
- this.setLastUpdate(byteBuffer.getLong());
- // parent id
- this.setParentProcId(byteBuffer.getLong());
- // time out
- this.setTimeout(byteBuffer.getLong());
- // stack index
- int stackIndexesLen = byteBuffer.getInt();
- if (stackIndexesLen >= 0) {
- List<Integer> indexList = new ArrayList<>(stackIndexesLen);
- for (int i = 0; i < stackIndexesLen; i++) {
- indexList.add(byteBuffer.getInt());
- }
- this.setStackIndexes(indexList);
- }
-
- // exception
- if (byteBuffer.get() == 1) {
- deserializeTypeInfoForCompatibility(byteBuffer);
- int messageBytesLength = byteBuffer.getInt();
- String errMsg = null;
- if (messageBytesLength > 0) {
- byte[] messageBytes = new byte[messageBytesLength];
- byteBuffer.get(messageBytes);
- errMsg = new String(messageBytes, StandardCharsets.UTF_8);
- }
- setFailure(new ProcedureException(errMsg));
- }
-
- // result
- int resultLen = byteBuffer.getInt();
- if (resultLen > 0) {
- byte[] resultArr = new byte[resultLen];
- byteBuffer.get(resultArr);
- }
- // has lock
- if (byteBuffer.get() == 1) {
- this.lockedWhenLoading();
- }
- }
-
- /**
- * Deserialize class Name and load class
- *
- * @param byteBuffer bytebuffer
- * @return Procedure
- */
- @Deprecated
- public static void deserializeTypeInfoForCompatibility(ByteBuffer byteBuffer) {
- int classNameBytesLen = byteBuffer.getInt();
- byte[] classNameBytes = new byte[classNameBytesLen];
- byteBuffer.get(classNameBytes);
- }
-
- /**
- * Acquire a lock, user should override it if necessary.
- *
- * @param env environment
- * @return state of lock
- */
- protected ProcedureLockState acquireLock(Env env) {
- return ProcedureLockState.LOCK_ACQUIRED;
- }
-
- /**
- * Release a lock, user should override it if necessary.
- *
- * @param env env
- */
- protected void releaseLock(Env env) {
- // no op
- }
-
- /**
- * Used to keep procedure lock even when the procedure is yielded or suspended.
- *
- * @param env env
- * @return true if hold the lock
- */
- protected boolean holdLock(Env env) {
- return false;
- }
-
- /**
- * To make executor yield between each execution step to give other procedures a chance to run.
- *
- * @param env environment
- * @return return true if yield is allowed.
- */
- protected boolean isYieldAfterExecution(Env env) {
- return false;
- }
-
- // -------------------------Internal methods - called by the procedureExecutor------------------
- /**
- * Internal method called by the ProcedureExecutor that starts the user-level code execute().
- *
- * @param env execute environment
- * @return sub procedures
- */
- protected Procedure<Env>[] doExecute(Env env) throws InterruptedException {
- try {
- updateTimestamp();
- return execute(env);
- } finally {
- updateTimestamp();
- }
- }
-
- /**
- * Internal method called by the ProcedureExecutor that starts the user-level code rollback().
- *
- * @param env execute environment
- * @throws IOException ioe
- * @throws InterruptedException interrupted exception
- */
- public void doRollback(Env env) throws IOException, InterruptedException, ProcedureException {
- try {
- updateTimestamp();
- rollback(env);
- } finally {
- updateTimestamp();
- }
- }
-
- /**
- * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock().
- *
- * @param env environment
- * @param store ProcedureStore
- * @return ProcedureLockState
- */
- public final ProcedureLockState doAcquireLock(Env env, IProcedureStore store) {
- if (lockedWhenLoading) {
- lockedWhenLoading = false;
- locked = true;
- return ProcedureLockState.LOCK_ACQUIRED;
- }
- ProcedureLockState state = acquireLock(env);
- if (state == ProcedureLockState.LOCK_ACQUIRED) {
- locked = true;
- store.update(this);
- }
- return state;
- }
-
- /**
- * Presist lock state of the procedure
- *
- * @param env environment
- * @param store ProcedureStore
- */
- public final void doReleaseLock(Env env, IProcedureStore store) {
- locked = false;
- if (getState() != ProcedureState.ROLLEDBACK) {
- store.update(this);
- }
- releaseLock(env);
- }
-
- public final void restoreLock(Env env) {
- if (!lockedWhenLoading) {
- LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock", this);
- return;
- }
- if (isFinished()) {
- LOG.debug("{} is already bypassed, skip acquiring lock.", this);
- return;
- }
- if (getState() == ProcedureState.WAITING && !holdLock(env)) {
- LOG.debug("{} is in WAITING STATE, and holdLock= false , skip acquiring lock.", this);
- return;
- }
- LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this);
- acquireLock(env);
- }
-
- @Override
- public String toString() {
- // Return the simple String presentation of the procedure.
- return toStringSimpleSB().toString();
- }
-
- /**
- * Build the StringBuilder for the simple form of procedure string.
- *
- * @return the StringBuilder
- */
- protected StringBuilder toStringSimpleSB() {
- final StringBuilder sb = new StringBuilder();
-
- sb.append("pid=");
- sb.append(getProcId());
-
- if (hasParent()) {
- sb.append(", ppid=");
- sb.append(getParentProcId());
- }
-
- /*
- * TODO
- * Enable later when this is being used.
- * Currently owner not used.
- if (hasOwner()) {
- sb.append(", owner=");
- sb.append(getOwner());
- }*/
-
- sb.append(", state="); // pState for Procedure State as opposed to any other kind.
- toStringState(sb);
-
- // Only print out locked if actually locked. Most of the time it is not.
- if (this.locked) {
- sb.append(", locked=").append(locked);
- }
- if (hasException()) {
- sb.append(", exception=" + getException());
- }
-
- sb.append("; ");
- toStringClassDetails(sb);
-
- return sb;
- }
-
- /** Extend the toString() information with more procedure details */
- public String toStringDetails() {
- final StringBuilder sb = toStringSimpleSB();
-
- sb.append(" submittedTime=");
- sb.append(getSubmittedTime());
-
- sb.append(", lastUpdate=");
- sb.append(getLastUpdate());
-
- final int[] stackIndices = getStackIndexes();
- if (stackIndices != null) {
- sb.append("\n");
- sb.append("stackIndexes=");
- sb.append(Arrays.toString(stackIndices));
- }
-
- return sb.toString();
- }
-
- /**
- * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating
- * generic Procedure State with Procedure particulars.
- *
- * @param builder Append current {@link ProcedureState}
- */
- protected void toStringState(StringBuilder builder) {
- builder.append(getState());
- }
-
- /**
- * Extend the toString() information with the procedure details e.g. className and parameters
- *
- * @param builder the string builder to use to append the proc specific information
- */
- protected void toStringClassDetails(StringBuilder builder) {
- builder.append(getClass().getName());
- }
-
- // ==========================================================================
- // Those fields are unchanged after initialization.
- //
- // Each procedure will get created from the user or during
- // ProcedureExecutor.start() during the load() phase and then submitted
- // to the executor. these fields will never be changed after initialization
- // ==========================================================================
- public long getProcId() {
- return procId;
- }
-
- public boolean hasParent() {
- return parentProcId != NO_PROC_ID;
- }
-
- public long getParentProcId() {
- return parentProcId;
- }
-
- public long getRootProcId() {
- return rootProcId;
- }
-
- public String getProcType() {
- return getClass().getSimpleName();
- }
-
- public long getSubmittedTime() {
- return submittedTime;
- }
-
- /** Called by the ProcedureExecutor to assign the ID to the newly created procedure. */
- public void setProcId(long procId) {
- this.procId = procId;
- }
-
- public void setProcRunnable() {
- this.submittedTime = System.currentTimeMillis();
- setState(ProcedureState.RUNNABLE);
- }
-
- /** Called by the ProcedureExecutor to assign the parent to the newly created procedure. */
- protected void setParentProcId(long parentProcId) {
- this.parentProcId = parentProcId;
- }
-
- protected void setRootProcId(long rootProcId) {
- this.rootProcId = rootProcId;
- }
-
- /**
- * Called on store load to initialize the Procedure internals after the creation/deserialization.
- */
- protected void setSubmittedTime(long submittedTime) {
- this.submittedTime = submittedTime;
- }
-
- // ==========================================================================
- // runtime state - timeout related
- // ==========================================================================
- /**
- * @param timeout timeout interval in msec
- */
- protected void setTimeout(long timeout) {
- this.timeout = timeout;
- }
-
- public boolean hasTimeout() {
- return timeout != NO_TIMEOUT;
- }
-
- /**
- * @return the timeout in msec
- */
- public long getTimeout() {
- return timeout;
- }
-
- /**
- * Called on store load to initialize the Procedure internals after the creation/deserialization.
- */
- protected void setLastUpdate(long lastUpdate) {
- this.lastUpdate = lastUpdate;
- }
-
- /** Called by ProcedureExecutor after each time a procedure step is executed. */
- protected void updateTimestamp() {
- this.lastUpdate = System.currentTimeMillis();
- }
-
- public long getLastUpdate() {
- return lastUpdate;
- }
-
- /**
- * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set
- * and the procedure is in the waiting queue.
- *
- * @return the timestamp of the next timeout.
- */
- protected long getTimeoutTimestamp() {
- return getLastUpdate() + getTimeout();
- }
-
- // ==========================================================================
- // runtime state
- // ==========================================================================
- /**
- * @return the time elapsed between the last update and the start time of the procedure.
- */
- public long elapsedTime() {
- return getLastUpdate() - getSubmittedTime();
- }
-
- /**
- * @return the serialized result if any, otherwise null
- */
- public byte[] getResult() {
- return result.get();
- }
-
- /**
- * The procedure may leave a "result" on completion.
- *
- * @param result the serialized result that will be passed to the client
- */
- protected void setResult(byte[] result) {
- this.result.set(result);
- }
-
- /**
- * Will only be called when loading procedures from procedure store, where we need to record
- * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)}
- * to actually acquire the lock.
- */
- final void lockedWhenLoading() {
- this.lockedWhenLoading = true;
- }
-
- /**
- * Can only be called when restarting, before the procedure actually being executed, as after we
- * actually call the {@link #doAcquireLock(Object, IProcedureStore)} method, we will reset {@link
- * #lockedWhenLoading} to false.
- *
- * <p>Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure
- * in front of a queue.
- */
- public boolean isLockedWhenLoading() {
- return lockedWhenLoading;
- }
-
- // ==============================================================================================
- // Runtime state, updated every operation by the ProcedureExecutor
- //
- // There is always 1 thread at the time operating on the state of the procedure.
- // The ProcedureExecutor may check and set states, or some Procecedure may
- // update its own state. but no concurrent updates. we use synchronized here
- // just because the procedure can get scheduled on different executor threads on each step.
- // ==============================================================================================
-
- /**
- * @return true if the procedure is in a RUNNABLE state.
- */
- public synchronized boolean isRunnable() {
- return state == ProcedureState.RUNNABLE;
- }
-
- public synchronized boolean isInitializing() {
- return state == ProcedureState.INITIALIZING;
- }
-
- /**
- * @return true if the procedure has failed. It may or may not have rolled back.
- */
- public synchronized boolean isFailed() {
- return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK;
- }
-
- /**
- * @return true if the procedure is finished successfully.
- */
- public synchronized boolean isSuccess() {
- return state == ProcedureState.SUCCESS && !hasException();
- }
-
- /**
- * @return true if the procedure is finished. The Procedure may be completed successfully or
- * rolledback.
- */
- public synchronized boolean isFinished() {
- return isSuccess() || state == ProcedureState.ROLLEDBACK;
- }
-
- /**
- * @return true if the procedure is waiting for a child to finish or for an external event.
- */
- public synchronized boolean isWaiting() {
- switch (state) {
- case WAITING:
- case WAITING_TIMEOUT:
- return true;
- default:
- break;
- }
- return false;
- }
-
- protected synchronized void setState(final ProcedureState state) {
- this.state = state;
- updateTimestamp();
- }
-
- public synchronized ProcedureState getState() {
- return state;
- }
-
- protected synchronized void setFailure(final String source, final Throwable cause) {
- setFailure(new ProcedureException(source, cause));
- }
-
- protected synchronized void setFailure(final ProcedureException exception) {
- this.exception = exception;
- if (!isFinished()) {
- setState(ProcedureState.FAILED);
- }
- }
-
- /**
- * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired.
- *
- * @return true to let the framework handle the timeout as abort, false in case the procedure
- * handled the timeout itself.
- */
- protected synchronized boolean setTimeoutFailure(Env env) {
- if (state == ProcedureState.WAITING_TIMEOUT) {
- long timeDiff = System.currentTimeMillis() - lastUpdate;
- setFailure(
- "ProcedureExecutor",
- new ProcedureException("Operation timed out after " + timeDiff + " ms."));
- return true;
- }
- return false;
- }
-
- public synchronized boolean hasException() {
- return exception != null;
- }
-
- public synchronized ProcedureException getException() {
- return exception;
- }
-
- /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
- protected synchronized void setChildrenLatch(int numChildren) {
- this.childrenLatch = numChildren;
- if (LOG.isTraceEnabled()) {
- LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString()));
- }
- }
-
- /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
- protected synchronized void incChildrenLatch() {
- // TODO: can this be inferred from the stack? I think so...
- this.childrenLatch++;
- if (LOG.isTraceEnabled()) {
- LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString()));
- }
- }
-
- /** Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. */
- private synchronized boolean childrenCountDown() {
- assert childrenLatch > 0 : this;
- boolean b = --childrenLatch == 0;
- if (LOG.isTraceEnabled()) {
- LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString()));
- }
- return b;
- }
-
- /**
- * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done.
- *
- * @return True if we were able to move procedure to RUNNABLE state.
- */
- synchronized boolean tryRunnable() {
- // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT
- if (getState() == ProcedureState.WAITING && childrenCountDown()) {
- setState(ProcedureState.RUNNABLE);
- return true;
- } else {
- return false;
- }
- }
-
- protected synchronized boolean hasChildren() {
- return childrenLatch > 0;
- }
-
- protected synchronized int getChildrenLatch() {
- return childrenLatch;
- }
-
- /**
- * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index
- * positions.
- */
- protected synchronized void addStackIndex(final int index) {
- if (stackIndexes == null) {
- stackIndexes = new int[] {index};
- } else {
- int count = stackIndexes.length;
- stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
- stackIndexes[count] = index;
- }
- }
-
- protected synchronized boolean removeStackIndex() {
- if (stackIndexes != null && stackIndexes.length > 1) {
- stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
- return false;
- } else {
- stackIndexes = null;
- return true;
- }
- }
-
- /**
- * Called on store load to initialize the Procedure internals after the creation/deserialization.
- */
- protected synchronized void setStackIndexes(final List<Integer> stackIndexes) {
- this.stackIndexes = new int[stackIndexes.size()];
- for (int i = 0; i < this.stackIndexes.length; ++i) {
- this.stackIndexes[i] = stackIndexes.get(i);
- }
- }
-
- protected synchronized boolean wasExecuted() {
- return stackIndexes != null;
- }
-
- protected synchronized int[] getStackIndexes() {
- return stackIndexes;
- }
-
- public void setRootProcedureId(long rootProcedureId) {
- this.rootProcId = rootProcedureId;
- }
-
- /**
- * @param a the first procedure to be compared.
- * @param b the second procedure to be compared.
- * @return true if the two procedures have the same parent
- */
- public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) {
- return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId());
- }
-
- @Override
- public int compareTo(Procedure<Env> other) {
- return Long.compare(getProcId(), other.getProcId());
- }
-
- /**
- * This function will be called just when procedure is submitted for execution.
- *
- * @param env The environment passed to the procedure executor
- */
- protected void updateMetricsOnSubmit(Env env) {
- if (env instanceof ConfigNodeProcedureEnv) {
- ((ConfigNodeProcedureEnv) env)
- .getConfigManager()
- .getProcedureManager()
- .getProcedureMetrics()
- .updateMetricsOnSubmit(getProcType());
- }
- }
-
- /**
- * This function will be called just after procedure execution is finished. Override this method
- * to update metrics at the end of the procedure. The default implementation adds runtime of a
- * procedure to a time histogram for successfully completed procedures. Increments failed counter
- * for failed procedures.
- *
- * @param env The environment passed to the procedure executor
- * @param runtime Runtime of the procedure in milliseconds
- * @param success true if procedure is completed successfully
- */
- protected void updateMetricsOnFinish(Env env, long runtime, boolean success) {
- if (env instanceof ConfigNodeProcedureEnv) {
- ((ConfigNodeProcedureEnv) env)
- .getConfigManager()
- .getProcedureManager()
- .getProcedureMetrics()
- .updateMetricsOnFinish(getProcType(), runtime, success);
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+1import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
+1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.io.DataOutputStream;
+1import java.io.IOException;
+1import java.nio.ByteBuffer;
+1import java.nio.charset.StandardCharsets;
+1import java.util.ArrayList;
+1import java.util.Arrays;
+1import java.util.List;
+1import java.util.concurrent.atomic.AtomicReference;
+1
+1/**
+1 * Abstract class of all procedures.
+1 *
+1 * @param <Env>
+1 */
+1public abstract class Procedure<Env> implements Comparable<Procedure<Env>> {
+1 private static final Logger LOG = LoggerFactory.getLogger(Procedure.class);
+1 public static final long NO_PROC_ID = -1;
+1 public static final long NO_TIMEOUT = -1;
+1
+1 private long parentProcId = NO_PROC_ID;
+1 private long rootProcId = NO_PROC_ID;
+1 private long procId = NO_PROC_ID;
+1 private long submittedTime;
+1
+1 private ProcedureState state = ProcedureState.INITIALIZING;
+1 private int childrenLatch = 0;
+1 private ProcedureException exception;
+1
+1 private volatile long timeout = NO_TIMEOUT;
+1 private volatile long lastUpdate;
+1
+1 private final AtomicReference<byte[]> result = new AtomicReference<>();
+1 private volatile boolean locked = false;
+1 private boolean lockedWhenLoading = false;
+1
+1 private int[] stackIndexes = null;
+1
+1 public final boolean hasLock() {
+1 return locked;
+1 }
+1
+1 // User level code, override it if necessary
+1
+1 /**
+1 * The main code of the procedure. It must be idempotent since execute() may be called multiple
+1 * times in case of machine failure in the middle of the execution.
+1 *
+1 * @param env the environment passed to the ProcedureExecutor
+1 * @return a set of sub-procedures to run or ourselves if there is more work to do or null if the
+1 * procedure is done.
+1 * @throws InterruptedException the procedure will be added back to the queue and retried later.
+1 */
+1 protected abstract Procedure<Env>[] execute(Env env) throws InterruptedException;
+1
+1 /**
+1 * The code to undo what was done by the execute() code. It is called when the procedure or one of
+1 * the sub-procedures failed or an abort was requested. It should cleanup all the resources
+1 * created by the execute() call. The implementation must be idempotent since rollback() may be
+1 * called multiple time in case of machine failure in the middle of the execution.
+1 *
+1 * @param env the environment passed to the ProcedureExecutor
+1 * @throws IOException temporary failure, the rollback will retry later
+1 * @throws InterruptedException the procedure will be added back to the queue and retried later
+1 */
+1 protected abstract void rollback(Env env)
+1 throws IOException, InterruptedException, ProcedureException;
+1
+1 public void serialize(DataOutputStream stream) throws IOException {
+1 // procid
+1 stream.writeLong(this.procId);
+1 // state
+1 stream.writeInt(this.state.ordinal());
+1 // submit time
+1 stream.writeLong(this.submittedTime);
+1 // last updated
+1 stream.writeLong(this.lastUpdate);
+1 // parent id
+1 stream.writeLong(this.parentProcId);
+1 // time out
+1 stream.writeLong(this.timeout);
+1 // stack indexes
+1 if (stackIndexes != null) {
+1 stream.writeInt(stackIndexes.length);
+1 for (int index : stackIndexes) {
+1 stream.writeInt(index);
+1 }
+1 } else {
+1 stream.writeInt(-1);
+1 }
+1
+1 // exceptions
+1 if (hasException()) {
+1 // symbol of exception
+1 stream.write((byte) 1);
+1 // exception's name
+1 String exceptionClassName = exception.getClass().getName();
+1 byte[] exceptionClassNameBytes = exceptionClassName.getBytes(StandardCharsets.UTF_8);
+1 stream.writeInt(exceptionClassNameBytes.length);
+1 stream.write(exceptionClassNameBytes);
+1 // exception's message
+1 String message = this.exception.getMessage();
+1 if (message != null) {
+1 byte[] messageBytes = message.getBytes(StandardCharsets.UTF_8);
+1 stream.writeInt(messageBytes.length);
+1 stream.write(messageBytes);
+1 } else {
+1 stream.writeInt(-1);
+1 }
+1 } else {
+1 // symbol of no exception
+1 stream.write((byte) 0);
+1 }
+1
+1 // result
+1 if (result.get() != null) {
+1 stream.writeInt(result.get().length);
+1 stream.write(result.get());
+1 } else {
+1 stream.writeInt(-1);
+1 }
+1
+1 // Has lock
+1 stream.write(this.hasLock() ? (byte) 1 : (byte) 0);
+1 }
+1
+1 public void deserialize(ByteBuffer byteBuffer) {
+1 // procid
+1 this.setProcId(byteBuffer.getLong());
+1 // state
+1 this.setState(ProcedureState.values()[byteBuffer.getInt()]);
+1 // submit time
+1 this.setSubmittedTime(byteBuffer.getLong());
+1 // last updated
+1 this.setLastUpdate(byteBuffer.getLong());
+1 // parent id
+1 this.setParentProcId(byteBuffer.getLong());
+1 // time out
+1 this.setTimeout(byteBuffer.getLong());
+1 // stack index
+1 int stackIndexesLen = byteBuffer.getInt();
+1 if (stackIndexesLen >= 0) {
+1 List<Integer> indexList = new ArrayList<>(stackIndexesLen);
+1 for (int i = 0; i < stackIndexesLen; i++) {
+1 indexList.add(byteBuffer.getInt());
+1 }
+1 this.setStackIndexes(indexList);
+1 }
+1
+1 // exception
+1 if (byteBuffer.get() == 1) {
+1 deserializeTypeInfoForCompatibility(byteBuffer);
+1 int messageBytesLength = byteBuffer.getInt();
+1 String errMsg = null;
+1 if (messageBytesLength > 0) {
+1 byte[] messageBytes = new byte[messageBytesLength];
+1 byteBuffer.get(messageBytes);
+1 errMsg = new String(messageBytes, StandardCharsets.UTF_8);
+1 }
+1 setFailure(new ProcedureException(errMsg));
+1 }
+1
+1 // result
+1 int resultLen = byteBuffer.getInt();
+1 if (resultLen > 0) {
+1 byte[] resultArr = new byte[resultLen];
+1 byteBuffer.get(resultArr);
+1 }
+1 // has lock
+1 if (byteBuffer.get() == 1) {
+1 this.lockedWhenLoading();
+1 }
+1 }
+1
+1 /**
+1 * Deserialize class Name and load class
+1 *
+1 * @param byteBuffer bytebuffer
+1 * @return Procedure
+1 */
+1 @Deprecated
+1 public static void deserializeTypeInfoForCompatibility(ByteBuffer byteBuffer) {
+1 int classNameBytesLen = byteBuffer.getInt();
+1 byte[] classNameBytes = new byte[classNameBytesLen];
+1 byteBuffer.get(classNameBytes);
+1 }
+1
+1 /**
+1 * Acquire a lock, user should override it if necessary.
+1 *
+1 * @param env environment
+1 * @return state of lock
+1 */
+1 protected ProcedureLockState acquireLock(Env env) {
+1 return ProcedureLockState.LOCK_ACQUIRED;
+1 }
+1
+1 /**
+1 * Release a lock, user should override it if necessary.
+1 *
+1 * @param env env
+1 */
+1 protected void releaseLock(Env env) {
+1 // no op
+1 }
+1
+1 /**
+1 * Used to keep procedure lock even when the procedure is yielded or suspended.
+1 *
+1 * @param env env
+1 * @return true if hold the lock
+1 */
+1 protected boolean holdLock(Env env) {
+1 return false;
+1 }
+1
+1 /**
+1 * To make executor yield between each execution step to give other procedures a chance to run.
+1 *
+1 * @param env environment
+1 * @return return true if yield is allowed.
+1 */
+1 protected boolean isYieldAfterExecution(Env env) {
+1 return false;
+1 }
+1
+1 // -------------------------Internal methods - called by the procedureExecutor------------------
+1 /**
+1 * Internal method called by the ProcedureExecutor that starts the user-level code execute().
+1 *
+1 * @param env execute environment
+1 * @return sub procedures
+1 */
+1 protected Procedure<Env>[] doExecute(Env env) throws InterruptedException {
+1 try {
+1 updateTimestamp();
+1 return execute(env);
+1 } finally {
+1 updateTimestamp();
+1 }
+1 }
+1
+1 /**
+1 * Internal method called by the ProcedureExecutor that starts the user-level code rollback().
+1 *
+1 * @param env execute environment
+1 * @throws IOException ioe
+1 * @throws InterruptedException interrupted exception
+1 */
+1 public void doRollback(Env env) throws IOException, InterruptedException, ProcedureException {
+1 try {
+1 updateTimestamp();
+1 rollback(env);
+1 } finally {
+1 updateTimestamp();
+1 }
+1 }
+1
+1 /**
+1 * Internal method called by the ProcedureExecutor that starts the user-level code acquireLock().
+1 *
+1 * @param env environment
+1 * @param store ProcedureStore
+1 * @return ProcedureLockState
+1 */
+1 public final ProcedureLockState doAcquireLock(Env env, IProcedureStore store) {
+1 if (lockedWhenLoading) {
+1 lockedWhenLoading = false;
+1 locked = true;
+1 return ProcedureLockState.LOCK_ACQUIRED;
+1 }
+1 ProcedureLockState state = acquireLock(env);
+1 if (state == ProcedureLockState.LOCK_ACQUIRED) {
+1 locked = true;
+1 store.update(this);
+1 }
+1 return state;
+1 }
+1
+1 /**
+1 * Presist lock state of the procedure
+1 *
+1 * @param env environment
+1 * @param store ProcedureStore
+1 */
+1 public final void doReleaseLock(Env env, IProcedureStore store) {
+1 locked = false;
+1 if (getState() != ProcedureState.ROLLEDBACK) {
+1 store.update(this);
+1 }
+1 releaseLock(env);
+1 }
+1
+1 public final void restoreLock(Env env) {
+1 if (!lockedWhenLoading) {
+1 LOG.debug("{} didn't hold the lock before restarting, skip acquiring lock", this);
+1 return;
+1 }
+1 if (isFinished()) {
+1 LOG.debug("{} is already bypassed, skip acquiring lock.", this);
+1 return;
+1 }
+1 if (getState() == ProcedureState.WAITING && !holdLock(env)) {
+1 LOG.debug("{} is in WAITING STATE, and holdLock= false , skip acquiring lock.", this);
+1 return;
+1 }
+1 LOG.debug("{} held the lock before restarting, call acquireLock to restore it.", this);
+1 acquireLock(env);
+1 }
+1
+1 @Override
+1 public String toString() {
+1 // Return the simple String presentation of the procedure.
+1 return toStringSimpleSB().toString();
+1 }
+1
+1 /**
+1 * Build the StringBuilder for the simple form of procedure string.
+1 *
+1 * @return the StringBuilder
+1 */
+1 protected StringBuilder toStringSimpleSB() {
+1 final StringBuilder sb = new StringBuilder();
+1
+1 sb.append("pid=");
+1 sb.append(getProcId());
+1
+1 if (hasParent()) {
+1 sb.append(", ppid=");
+1 sb.append(getParentProcId());
+1 }
+1
+1 /*
+1 * TODO
+1 * Enable later when this is being used.
+1 * Currently owner not used.
+1 if (hasOwner()) {
+1 sb.append(", owner=");
+1 sb.append(getOwner());
+1 }*/
+1
+1 sb.append(", state="); // pState for Procedure State as opposed to any other kind.
+1 toStringState(sb);
+1
+1 // Only print out locked if actually locked. Most of the time it is not.
+1 if (this.locked) {
+1 sb.append(", locked=").append(locked);
+1 }
+1 if (hasException()) {
+1 sb.append(", exception=" + getException());
+1 }
+1
+1 sb.append("; ");
+1 toStringClassDetails(sb);
+1
+1 return sb;
+1 }
+1
+1 /** Extend the toString() information with more procedure details */
+1 public String toStringDetails() {
+1 final StringBuilder sb = toStringSimpleSB();
+1
+1 sb.append(" submittedTime=");
+1 sb.append(getSubmittedTime());
+1
+1 sb.append(", lastUpdate=");
+1 sb.append(getLastUpdate());
+1
+1 final int[] stackIndices = getStackIndexes();
+1 if (stackIndices != null) {
+1 sb.append("\n");
+1 sb.append("stackIndexes=");
+1 sb.append(Arrays.toString(stackIndices));
+1 }
+1
+1 return sb.toString();
+1 }
+1
+1 /**
+1 * Called from {@link #toString()} when interpolating {@link Procedure} State. Allows decorating
+1 * generic Procedure State with Procedure particulars.
+1 *
+1 * @param builder Append current {@link ProcedureState}
+1 */
+1 protected void toStringState(StringBuilder builder) {
+1 builder.append(getState());
+1 }
+1
+1 /**
+1 * Extend the toString() information with the procedure details e.g. className and parameters
+1 *
+1 * @param builder the string builder to use to append the proc specific information
+1 */
+1 protected void toStringClassDetails(StringBuilder builder) {
+1 builder.append(getClass().getName());
+1 }
+1
+1 // ==========================================================================
+1 // Those fields are unchanged after initialization.
+1 //
+1 // Each procedure will get created from the user or during
+1 // ProcedureExecutor.start() during the load() phase and then submitted
+1 // to the executor. these fields will never be changed after initialization
+1 // ==========================================================================
+1 public long getProcId() {
+1 return procId;
+1 }
+1
+1 public boolean hasParent() {
+1 return parentProcId != NO_PROC_ID;
+1 }
+1
+1 public long getParentProcId() {
+1 return parentProcId;
+1 }
+1
+1 public long getRootProcId() {
+1 return rootProcId;
+1 }
+1
+1 public String getProcType() {
+1 return getClass().getSimpleName();
+1 }
+1
+1 public long getSubmittedTime() {
+1 return submittedTime;
+1 }
+1
+1 /** Called by the ProcedureExecutor to assign the ID to the newly created procedure. */
+1 public void setProcId(long procId) {
+1 this.procId = procId;
+1 }
+1
+1 public void setProcRunnable() {
+1 this.submittedTime = System.currentTimeMillis();
+1 setState(ProcedureState.RUNNABLE);
+1 }
+1
+1 /** Called by the ProcedureExecutor to assign the parent to the newly created procedure. */
+1 protected void setParentProcId(long parentProcId) {
+1 this.parentProcId = parentProcId;
+1 }
+1
+1 protected void setRootProcId(long rootProcId) {
+1 this.rootProcId = rootProcId;
+1 }
+1
+1 /**
+1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
+1 */
+1 protected void setSubmittedTime(long submittedTime) {
+1 this.submittedTime = submittedTime;
+1 }
+1
+1 // ==========================================================================
+1 // runtime state - timeout related
+1 // ==========================================================================
+1 /**
+1 * @param timeout timeout interval in msec
+1 */
+1 protected void setTimeout(long timeout) {
+1 this.timeout = timeout;
+1 }
+1
+1 public boolean hasTimeout() {
+1 return timeout != NO_TIMEOUT;
+1 }
+1
+1 /**
+1 * @return the timeout in msec
+1 */
+1 public long getTimeout() {
+1 return timeout;
+1 }
+1
+1 /**
+1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
+1 */
+1 protected void setLastUpdate(long lastUpdate) {
+1 this.lastUpdate = lastUpdate;
+1 }
+1
+1 /** Called by ProcedureExecutor after each time a procedure step is executed. */
+1 protected void updateTimestamp() {
+1 this.lastUpdate = System.currentTimeMillis();
+1 }
+1
+1 public long getLastUpdate() {
+1 return lastUpdate;
+1 }
+1
+1 /**
+1 * Timeout of the next timeout. Called by the ProcedureExecutor if the procedure has timeout set
+1 * and the procedure is in the waiting queue.
+1 *
+1 * @return the timestamp of the next timeout.
+1 */
+1 protected long getTimeoutTimestamp() {
+1 return getLastUpdate() + getTimeout();
+1 }
+1
+1 // ==========================================================================
+1 // runtime state
+1 // ==========================================================================
+1 /**
+1 * @return the time elapsed between the last update and the start time of the procedure.
+1 */
+1 public long elapsedTime() {
+1 return getLastUpdate() - getSubmittedTime();
+1 }
+1
+1 /**
+1 * @return the serialized result if any, otherwise null
+1 */
+1 public byte[] getResult() {
+1 return result.get();
+1 }
+1
+1 /**
+1 * The procedure may leave a "result" on completion.
+1 *
+1 * @param result the serialized result that will be passed to the client
+1 */
+1 protected void setResult(byte[] result) {
+1 this.result.set(result);
+1 }
+1
+1 /**
+1 * Will only be called when loading procedures from procedure store, where we need to record
+1 * whether the procedure has already held a lock. Later we will call {@link #restoreLock(Object)}
+1 * to actually acquire the lock.
+1 */
+1 final void lockedWhenLoading() {
+1 this.lockedWhenLoading = true;
+1 }
+1
+1 /**
+1 * Can only be called when restarting, before the procedure actually being executed, as after we
+1 * actually call the {@link #doAcquireLock(Object, IProcedureStore)} method, we will reset {@link
+1 * #lockedWhenLoading} to false.
+1 *
+1 * <p>Now it is only used in the ProcedureScheduler to determine whether we should put a Procedure
+1 * in front of a queue.
+1 */
+1 public boolean isLockedWhenLoading() {
+1 return lockedWhenLoading;
+1 }
+1
+1 // ==============================================================================================
+1 // Runtime state, updated every operation by the ProcedureExecutor
+1 //
+1 // There is always 1 thread at the time operating on the state of the procedure.
+1 // The ProcedureExecutor may check and set states, or some Procecedure may
+1 // update its own state. but no concurrent updates. we use synchronized here
+1 // just because the procedure can get scheduled on different executor threads on each step.
+1 // ==============================================================================================
+1
+1 /**
+1 * @return true if the procedure is in a RUNNABLE state.
+1 */
+1 public synchronized boolean isRunnable() {
+1 return state == ProcedureState.RUNNABLE;
+1 }
+1
+1 public synchronized boolean isInitializing() {
+1 return state == ProcedureState.INITIALIZING;
+1 }
+1
+1 /**
+1 * @return true if the procedure has failed. It may or may not have rolled back.
+1 */
+1 public synchronized boolean isFailed() {
+1 return state == ProcedureState.FAILED || state == ProcedureState.ROLLEDBACK;
+1 }
+1
+1 /**
+1 * @return true if the procedure is finished successfully.
+1 */
+1 public synchronized boolean isSuccess() {
+1 return state == ProcedureState.SUCCESS && !hasException();
+1 }
+1
+1 /**
+1 * @return true if the procedure is finished. The Procedure may be completed successfully or
+1 * rolledback.
+1 */
+1 public synchronized boolean isFinished() {
+1 return isSuccess() || state == ProcedureState.ROLLEDBACK;
+1 }
+1
+1 /**
+1 * @return true if the procedure is waiting for a child to finish or for an external event.
+1 */
+1 public synchronized boolean isWaiting() {
+1 switch (state) {
+1 case WAITING:
+1 case WAITING_TIMEOUT:
+1 return true;
+1 default:
+1 break;
+1 }
+1 return false;
+1 }
+1
+1 protected synchronized void setState(final ProcedureState state) {
+1 this.state = state;
+1 updateTimestamp();
+1 }
+1
+1 public synchronized ProcedureState getState() {
+1 return state;
+1 }
+1
+1 protected synchronized void setFailure(final String source, final Throwable cause) {
+1 setFailure(new ProcedureException(source, cause));
+1 }
+1
+1 protected synchronized void setFailure(final ProcedureException exception) {
+1 this.exception = exception;
+1 if (!isFinished()) {
+1 setState(ProcedureState.FAILED);
+1 }
+1 }
+1
+1 /**
+1 * Called by the ProcedureExecutor when the timeout set by setTimeout() is expired.
+1 *
+1 * @return true to let the framework handle the timeout as abort, false in case the procedure
+1 * handled the timeout itself.
+1 */
+1 protected synchronized boolean setTimeoutFailure(Env env) {
+1 if (state == ProcedureState.WAITING_TIMEOUT) {
+1 long timeDiff = System.currentTimeMillis() - lastUpdate;
+1 setFailure(
+1 "ProcedureExecutor",
+1 new ProcedureException("Operation timed out after " + timeDiff + " ms."));
+1 return true;
+1 }
+1 return false;
+1 }
+1
+1 public synchronized boolean hasException() {
+1 return exception != null;
+1 }
+1
+1 public synchronized ProcedureException getException() {
+1 return exception;
+1 }
+1
+1 /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+1 protected synchronized void setChildrenLatch(int numChildren) {
+1 this.childrenLatch = numChildren;
+1 if (LOG.isTraceEnabled()) {
+1 LOG.trace("CHILD LATCH INCREMENT SET " + this.childrenLatch, new Throwable(this.toString()));
+1 }
+1 }
+1
+1 /** Called by the ProcedureExecutor on procedure-load to restore the latch state */
+1 protected synchronized void incChildrenLatch() {
+1 // TODO: can this be inferred from the stack? I think so...
+1 this.childrenLatch++;
+1 if (LOG.isTraceEnabled()) {
+1 LOG.trace("CHILD LATCH INCREMENT " + this.childrenLatch, new Throwable(this.toString()));
+1 }
+1 }
+1
+1 /** Called by the ProcedureExecutor to notify that one of the sub-procedures has completed. */
+1 private synchronized boolean childrenCountDown() {
+1 assert childrenLatch > 0 : this;
+1 boolean b = --childrenLatch == 0;
+1 if (LOG.isTraceEnabled()) {
+1 LOG.trace("CHILD LATCH DECREMENT " + childrenLatch, new Throwable(this.toString()));
+1 }
+1 return b;
+1 }
+1
+1 /**
+1 * Try to set this procedure into RUNNABLE state. Succeeds if all subprocedures/children are done.
+1 *
+1 * @return True if we were able to move procedure to RUNNABLE state.
+1 */
+1 synchronized boolean tryRunnable() {
+1 // Don't use isWaiting in the below; it returns true for WAITING and WAITING_TIMEOUT
+1 if (getState() == ProcedureState.WAITING && childrenCountDown()) {
+1 setState(ProcedureState.RUNNABLE);
+1 return true;
+1 } else {
+1 return false;
+1 }
+1 }
+1
+1 protected synchronized boolean hasChildren() {
+1 return childrenLatch > 0;
+1 }
+1
+1 protected synchronized int getChildrenLatch() {
+1 return childrenLatch;
+1 }
+1
+1 /**
+1 * Called by the RootProcedureState on procedure execution. Each procedure store its stack-index
+1 * positions.
+1 */
+1 protected synchronized void addStackIndex(final int index) {
+1 if (stackIndexes == null) {
+1 stackIndexes = new int[] {index};
+1 } else {
+1 int count = stackIndexes.length;
+1 stackIndexes = Arrays.copyOf(stackIndexes, count + 1);
+1 stackIndexes[count] = index;
+1 }
+1 }
+1
+1 protected synchronized boolean removeStackIndex() {
+1 if (stackIndexes != null && stackIndexes.length > 1) {
+1 stackIndexes = Arrays.copyOf(stackIndexes, stackIndexes.length - 1);
+1 return false;
+1 } else {
+1 stackIndexes = null;
+1 return true;
+1 }
+1 }
+1
+1 /**
+1 * Called on store load to initialize the Procedure internals after the creation/deserialization.
+1 */
+1 protected synchronized void setStackIndexes(final List<Integer> stackIndexes) {
+1 this.stackIndexes = new int[stackIndexes.size()];
+1 for (int i = 0; i < this.stackIndexes.length; ++i) {
+1 this.stackIndexes[i] = stackIndexes.get(i);
+1 }
+1 }
+1
+1 protected synchronized boolean wasExecuted() {
+1 return stackIndexes != null;
+1 }
+1
+1 protected synchronized int[] getStackIndexes() {
+1 return stackIndexes;
+1 }
+1
+1 public void setRootProcedureId(long rootProcedureId) {
+1 this.rootProcId = rootProcedureId;
+1 }
+1
+1 /**
+1 * @param a the first procedure to be compared.
+1 * @param b the second procedure to be compared.
+1 * @return true if the two procedures have the same parent
+1 */
+1 public static boolean haveSameParent(Procedure<?> a, Procedure<?> b) {
+1 return a.hasParent() && b.hasParent() && (a.getParentProcId() == b.getParentProcId());
+1 }
+1
+1 @Override
+1 public int compareTo(Procedure<Env> other) {
+1 return Long.compare(getProcId(), other.getProcId());
+1 }
+1
+1 /**
+1 * This function will be called just when procedure is submitted for execution.
+1 *
+1 * @param env The environment passed to the procedure executor
+1 */
+1 protected void updateMetricsOnSubmit(Env env) {
+1 if (env instanceof ConfigNodeProcedureEnv) {
+1 ((ConfigNodeProcedureEnv) env)
+1 .getConfigManager()
+1 .getProcedureManager()
+1 .getProcedureMetrics()
+1 .updateMetricsOnSubmit(getProcType());
+1 }
+1 }
+1
+1 /**
+1 * This function will be called just after procedure execution is finished. Override this method
+1 * to update metrics at the end of the procedure. The default implementation adds runtime of a
+1 * procedure to a time histogram for successfully completed procedures. Increments failed counter
+1 * for failed procedures.
+1 *
+1 * @param env The environment passed to the procedure executor
+1 * @param runtime Runtime of the procedure in milliseconds
+1 * @param success true if procedure is completed successfully
+1 */
+1 protected void updateMetricsOnFinish(Env env, long runtime, boolean success) {
+1 if (env instanceof ConfigNodeProcedureEnv) {
+1 ((ConfigNodeProcedureEnv) env)
+1 .getConfigManager()
+1 .getProcedureManager()
+1 .getProcedureMetrics()
+1 .updateMetricsOnFinish(getProcType(), runtime, success);
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
index 0d8368583b4..4c7421885f7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureExecutor.java
@@ -1,940 +1,941 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.commons.concurrent.ThreadName;
-import org.apache.iotdb.commons.utils.TestOnly;
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
-import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
-import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
-
-import com.google.common.base.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Deque;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static org.apache.iotdb.confignode.procedure.Procedure.NO_PROC_ID;
-
-public class ProcedureExecutor<Env> {
- private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);
-
- private final ConcurrentHashMap<Long, CompletedProcedureContainer<Env>> completed =
- new ConcurrentHashMap<>();
-
- private final ConcurrentHashMap<Long, RootProcedureStack<Env>> rollbackStack =
- new ConcurrentHashMap<>();
-
- private final ConcurrentHashMap<Long, Procedure<Env>> procedures = new ConcurrentHashMap<>();
-
- private ThreadGroup threadGroup;
-
- private CopyOnWriteArrayList<WorkerThread> workerThreads;
-
- private TimeoutExecutorThread<Env> timeoutExecutor;
-
- private TimeoutExecutorThread<Env> workerMonitorExecutor;
-
- private int corePoolSize;
- private int maxPoolSize;
-
- private final ProcedureScheduler scheduler;
-
- private final AtomicLong workId = new AtomicLong(0);
- private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
- private final AtomicBoolean running = new AtomicBoolean(false);
- private final Env environment;
- private final IProcedureStore<Env> store;
-
- public ProcedureExecutor(
- final Env environment, final IProcedureStore<Env> store, final ProcedureScheduler scheduler) {
- this.environment = environment;
- this.scheduler = scheduler;
- this.store = store;
- }
-
- @TestOnly
- public ProcedureExecutor(final Env environment, final IProcedureStore<Env> store) {
- this(environment, store, new SimpleProcedureScheduler());
- }
-
- public void init(int numThreads) {
- this.corePoolSize = numThreads;
- this.maxPoolSize = 10 * numThreads;
- this.threadGroup = new ThreadGroup(ThreadName.CONFIG_NODE_PROCEDURE_WORKER.getName());
- this.timeoutExecutor =
- new TimeoutExecutorThread<>(
- this, threadGroup, ThreadName.CONFIG_NODE_TIMEOUT_EXECUTOR.getName());
- this.workerMonitorExecutor =
- new TimeoutExecutorThread<>(
- this, threadGroup, ThreadName.CONFIG_NODE_WORKER_THREAD_MONITOR.getName());
- workId.set(0);
- workerThreads = new CopyOnWriteArrayList<>();
- for (int i = 0; i < corePoolSize; i++) {
- workerThreads.add(new WorkerThread(threadGroup));
- }
- // Add worker monitor
- workerMonitorExecutor.add(new WorkerMonitor());
-
- scheduler.start();
- recover();
- }
-
- private void recover() {
- // 1.Build rollback stack
- List<Procedure<Env>> procedureList = getProcedureListFromDifferentVersion();
- // Load procedure wal file
- for (Procedure<Env> proc : procedureList) {
- if (proc.isFinished()) {
- completed.putIfAbsent(proc.getProcId(), new CompletedProcedureContainer<>(proc));
- } else {
- if (!proc.hasParent()) {
- rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
- }
- procedures.putIfAbsent(proc.getProcId(), proc);
- }
- }
- List<Procedure<Env>> runnableList = new ArrayList<>();
- List<Procedure<Env>> failedList = new ArrayList<>();
- List<Procedure<Env>> waitingList = new ArrayList<>();
- List<Procedure<Env>> waitingTimeoutList = new ArrayList<>();
- for (Procedure<Env> proc : procedureList) {
- if (proc.isFinished() && !proc.hasParent()) {
- continue;
- }
- long rootProcedureId = getRootProcedureId(proc);
- if (proc.hasParent()) {
- Procedure<Env> parent = procedures.get(proc.getParentProcId());
- if (parent != null && !proc.isFinished()) {
- parent.incChildrenLatch();
- }
- }
- RootProcedureStack<Env> rootStack = rollbackStack.get(rootProcedureId);
- if (rootStack != null) {
- rootStack.loadStack(proc);
- }
- proc.setRootProcedureId(rootProcedureId);
- switch (proc.getState()) {
- case RUNNABLE:
- runnableList.add(proc);
- break;
- case FAILED:
- failedList.add(proc);
- break;
- case WAITING:
- waitingList.add(proc);
- break;
- case WAITING_TIMEOUT:
- waitingTimeoutList.add(proc);
- break;
- case ROLLEDBACK:
- case INITIALIZING:
- LOG.error("Unexpected state:{} for {}", proc.getState(), proc);
- throw new UnsupportedOperationException("Unexpected state");
- default:
- break;
- }
- }
-
- waitingList.forEach(
- procedure -> {
- if (!procedure.hasChildren()) {
- // Normally, WAITING procedures should be wakened by its children.
- // But, there is a case that, all the children are successful, and before
- // they can wake up their parent procedure, the master was killed.
- // So, during recovering the procedures from ProcedureWal, its children
- // are not loaded because of their SUCCESS state.
- // So we need to continue to run this WAITING procedure. Before
- // executing, we need to set its state to RUNNABLE.
- procedure.setState(ProcedureState.RUNNABLE);
- runnableList.add(procedure);
- }
- });
- restoreLocks();
-
- waitingTimeoutList.forEach(timeoutExecutor::add);
-
- failedList.forEach(scheduler::addBack);
- runnableList.forEach(scheduler::addBack);
- scheduler.signalAll();
- }
-
- private List<Procedure<Env>> getProcedureListFromDifferentVersion() {
- if (store.isOldVersionProcedureStore()) {
- LOG.info("Old procedure directory detected, upgrade beginning...");
- return store.load();
- } else {
- return store.getProcedures();
- }
- }
-
- /**
- * Helper to look up the root Procedure ID.
- *
- * @param proc given a specified procedure.
- */
- Long getRootProcedureId(Procedure<Env> proc) {
- while (proc.hasParent()) {
- proc = procedures.get(proc.getParentProcId());
- if (proc == null) {
- return NO_PROC_ID;
- }
- }
- return proc.getProcId();
- }
-
- private void releaseLock(Procedure<Env> procedure, boolean force) {
- if (force || !procedure.holdLock(this.environment) || procedure.isFinished()) {
- procedure.doReleaseLock(this.environment, store);
- }
- }
-
- private void restoreLock(Procedure procedure, Set<Long> restored) {
- procedure.restoreLock(environment);
- restored.add(procedure.getProcId());
- }
-
- private void restoreLocks(Deque<Procedure<Env>> stack, Set<Long> restored) {
- while (!stack.isEmpty()) {
- restoreLock(stack.pop(), restored);
- }
- }
-
- private void restoreLocks() {
- Set<Long> restored = new HashSet<>();
- Deque<Procedure<Env>> stack = new ArrayDeque<>();
- procedures
- .values()
- .forEach(
- procedure -> {
- while (procedure != null) {
- if (restored.contains(procedure.getProcId())) {
- restoreLocks(stack, restored);
- return;
- }
- if (!procedure.hasParent()) {
- restoreLock(procedure, restored);
- restoreLocks(stack, restored);
- return;
- }
- stack.push(procedure);
- procedure = procedures.get(procedure.getParentProcId());
- }
- });
- }
-
- public void startWorkers() {
- if (!running.compareAndSet(false, true)) {
- LOG.warn("Already running");
- return;
- }
- timeoutExecutor.start();
- workerMonitorExecutor.start();
- for (WorkerThread workerThread : workerThreads) {
- workerThread.start();
- }
- LOG.info("{} procedure workers are started.", workerThreads.size());
- }
-
- public void startCompletedCleaner(long cleanTimeInterval, long cleanEvictTTL) {
- addInternalProcedure(
- new CompletedProcedureRecycler(store, completed, cleanTimeInterval, cleanEvictTTL));
- }
-
- public void addInternalProcedure(InternalProcedure interalProcedure) {
- if (interalProcedure == null) {
- return;
- }
- interalProcedure.setState(ProcedureState.WAITING_TIMEOUT);
- timeoutExecutor.add(interalProcedure);
- }
-
- public boolean removeInternalProcedure(InternalProcedure internalProcedure) {
- if (internalProcedure == null) {
- return true;
- }
- internalProcedure.setState(ProcedureState.SUCCESS);
- return timeoutExecutor.remove(internalProcedure);
- }
-
- /**
- * Executes procedure
- *
- * <p>Calls doExecute() if success and return subprocedures submit sub procs set the state to
- * WAITING, wait for all sub procs completed. else if no sub procs procedure completed
- * successfully set procedure's parent to RUNNABLE in case of failure start rollback of the
- * procedure.
- *
- * @param proc procedure
- */
- private void executeProcedure(Procedure<Env> proc) {
- if (proc.isFinished()) {
- LOG.debug("{} is already finished.", proc);
- return;
- }
- final Long rootProcId = getRootProcedureId(proc);
- if (rootProcId == null) {
- LOG.warn("Rollback because parent is done/rolledback, proc is {}", proc);
- executeRollback(proc);
- return;
- }
- RootProcedureStack<Env> rootProcStack = rollbackStack.get(rootProcId);
- if (rootProcStack == null) {
- LOG.warn("Rollback stack is null for {}", proc.getProcId());
- return;
- }
- ProcedureLockState lockState = null;
- try {
- do {
- if (!rootProcStack.acquire()) {
- if (rootProcStack.setRollback()) {
- lockState = executeRootStackRollback(rootProcId, rootProcStack);
- switch (lockState) {
- case LOCK_ACQUIRED:
- break;
- case LOCK_EVENT_WAIT:
- LOG.info("LOCK_EVENT_WAIT rollback {}", proc);
- rootProcStack.unsetRollback();
- break;
- case LOCK_YIELD_WAIT:
- rootProcStack.unsetRollback();
- scheduler.yield(proc);
- break;
- default:
- throw new UnsupportedOperationException();
- }
- } else {
- if (!proc.wasExecuted()) {
- switch (executeRollback(proc)) {
- case LOCK_ACQUIRED:
- break;
- case LOCK_EVENT_WAIT:
- LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc);
- break;
- case LOCK_YIELD_WAIT:
- scheduler.yield(proc);
- break;
- default:
- throw new UnsupportedOperationException();
- }
- }
- }
- break;
- }
- lockState = acquireLock(proc);
- switch (lockState) {
- case LOCK_ACQUIRED:
- executeProcedure(rootProcStack, proc);
- break;
- case LOCK_YIELD_WAIT:
- case LOCK_EVENT_WAIT:
- LOG.info("{} lockstate is {}", proc, lockState);
- break;
- default:
- throw new UnsupportedOperationException();
- }
- rootProcStack.release();
-
- if (proc.isSuccess()) {
- // update metrics on finishing the procedure
- proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);
- LOG.debug("{} finished in {}ms successfully.", proc, proc.elapsedTime());
- if (proc.getProcId() == rootProcId) {
- rootProcedureCleanup(proc);
- } else {
- executeCompletionCleanup(proc);
- }
- return;
- }
-
- } while (rootProcStack.isFailed());
- } finally {
- // Only after procedure has completed execution can it be allowed to be rescheduled to prevent
- // data races
- if (Objects.equals(lockState, ProcedureLockState.LOCK_EVENT_WAIT)) {
- LOG.info("procedureId {} wait for lock.", proc.getProcId());
- ((ConfigNodeProcedureEnv) this.environment).getNodeLock().waitProcedure(proc);
- }
- }
- }
-
- /**
- * execute procedure and submit its children
- *
- * @param rootProcStack procedure's root proc stack
- * @param proc procedure
- */
- private void executeProcedure(RootProcedureStack rootProcStack, Procedure<Env> proc) {
- if (proc.getState() != ProcedureState.RUNNABLE) {
- LOG.error(
- "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}", proc);
- return;
- }
- boolean reExecute;
-
- Procedure<Env>[] subprocs = null;
- do {
- reExecute = false;
- try {
- subprocs = proc.doExecute(this.environment);
- if (subprocs != null && subprocs.length == 0) {
- subprocs = null;
- }
- } catch (InterruptedException e) {
- LOG.warn("Interrupt during execution, suspend or retry it later.", e);
- yieldProcedure(proc);
- } catch (Throwable e) {
- LOG.error("CODE-BUG:{}", proc, e);
- proc.setFailure(new ProcedureException(e.getMessage(), e));
- }
-
- if (!proc.isFailed()) {
- if (subprocs != null) {
- if (subprocs.length == 1 && subprocs[0] == proc) {
- subprocs = null;
- reExecute = true;
- } else {
- subprocs = initializeChildren(rootProcStack, proc, subprocs);
- LOG.info("Initialized sub procs:{}", Arrays.toString(subprocs));
- }
- } else if (proc.getState() == ProcedureState.WAITING_TIMEOUT) {
- LOG.info("Added into timeoutExecutor {}", proc);
- } else {
- proc.setState(ProcedureState.SUCCESS);
- }
- }
- // add procedure into rollback stack.
- rootProcStack.addRollbackStep(proc);
-
- updateStoreOnExecution(rootProcStack, proc, subprocs);
-
- if (!store.isRunning()) {
- return;
- }
-
- if (proc.isRunnable() && proc.isYieldAfterExecution(this.environment)) {
- yieldProcedure(proc);
- return;
- }
- } while (reExecute);
-
- if (subprocs != null && !proc.isFailed()) {
- submitChildrenProcedures(subprocs);
- }
-
- releaseLock(proc, false);
- if (proc.isFinished() && proc.hasParent()) {
- countDownChildren(rootProcStack, proc);
- }
- }
-
- /**
- * Serve as a countdown latch to check whether all children have already completed.
- *
- * @param rootProcStack root procedure stack
- * @param proc proc
- */
- private void countDownChildren(RootProcedureStack rootProcStack, Procedure<Env> proc) {
- Procedure<Env> parent = procedures.get(proc.getParentProcId());
- if (parent == null && rootProcStack.isRollingback()) {
- return;
- }
- if (parent != null && parent.tryRunnable()) {
- // If success, means all its children have completed, move parent to front of the queue.
- store.update(parent);
- scheduler.addFront(parent);
- LOG.info(
- "Finished subprocedure pid={}, resume processing ppid={}",
- proc.getProcId(),
- parent.getProcId());
- }
- }
-
- /**
- * Submit children procedures.
- *
- * @param subprocs children procedures
- */
- private void submitChildrenProcedures(Procedure<Env>[] subprocs) {
- for (Procedure<Env> subproc : subprocs) {
- subproc.updateMetricsOnSubmit(getEnvironment());
- procedures.put(subproc.getProcId(), subproc);
- scheduler.addFront(subproc);
- LOG.info("Sub-Procedure pid={} has been submitted", subproc.getProcId());
- }
- }
-
- private void updateStoreOnExecution(
- RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
- if (subprocs != null && !proc.isFailed()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stored {}, children {}", proc, Arrays.toString(subprocs));
- }
- store.update(subprocs);
- } else {
- LOG.debug("Store update {}", proc);
- if (proc.isFinished() && !proc.hasParent()) {
- final long[] childProcIds = rootProcStack.getSubprocedureIds();
- if (childProcIds != null) {
- store.delete(childProcIds);
- for (long childProcId : childProcIds) {
- procedures.remove(childProcId);
- }
- } else {
- store.update(proc);
- }
- } else {
- store.update(proc);
- }
- }
- }
-
- private Procedure<Env>[] initializeChildren(
- RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
- final long rootProcedureId = getRootProcedureId(proc);
- for (int i = 0; i < subprocs.length; i++) {
- Procedure<Env> subproc = subprocs[i];
- if (subproc == null) {
- String errMsg = "subproc[" + i + "] is null, aborting procedure";
- proc.setFailure(new ProcedureException((errMsg), new IllegalArgumentException(errMsg)));
- return null;
- }
- subproc.setParentProcId(proc.getProcId());
- subproc.setRootProcId(rootProcedureId);
- subproc.setProcId(store.getNextProcId());
- subproc.setProcRunnable();
- rootProcStack.addSubProcedure(subproc);
- }
-
- if (!proc.isFailed()) {
- proc.setChildrenLatch(subprocs.length);
- switch (proc.getState()) {
- case RUNNABLE:
- proc.setState(ProcedureState.WAITING);
- break;
- case WAITING_TIMEOUT:
- timeoutExecutor.add(proc);
- break;
- default:
- break;
- }
- }
- return subprocs;
- }
-
- private void yieldProcedure(Procedure<Env> proc) {
- releaseLock(proc, false);
- scheduler.yield(proc);
- }
-
- /**
- * Rollback full root procedure stack.
- *
- * @param rootProcId root procedure id
- * @param procedureStack root procedure stack
- * @return lock state
- */
- private ProcedureLockState executeRootStackRollback(
- Long rootProcId, RootProcedureStack procedureStack) {
- Procedure<Env> rootProcedure = procedures.get(rootProcId);
- ProcedureException exception = rootProcedure.getException();
- if (exception == null) {
- exception = procedureStack.getException();
- rootProcedure.setFailure(exception);
- store.update(rootProcedure);
- }
- List<Procedure<Env>> subprocStack = procedureStack.getSubproceduresStack();
- int stackTail = subprocStack.size();
- while (stackTail-- > 0) {
- Procedure<Env> procedure = subprocStack.get(stackTail);
- if (procedure.isSuccess()) {
- subprocStack.remove(stackTail);
- cleanupAfterRollback(procedure);
- continue;
- }
- ProcedureLockState lockState = acquireLock(procedure);
- if (lockState != ProcedureLockState.LOCK_ACQUIRED) {
- return lockState;
- }
- lockState = executeRollback(procedure);
- releaseLock(procedure, false);
-
- boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED;
- abortRollback |= !isRunning() || !store.isRunning();
- if (abortRollback) {
- return lockState;
- }
-
- if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) {
- return ProcedureLockState.LOCK_YIELD_WAIT;
- }
-
- if (procedure != rootProcedure) {
- executeCompletionCleanup(procedure);
- }
- }
-
- LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime());
- rootProcedureCleanup(rootProcedure);
- return ProcedureLockState.LOCK_ACQUIRED;
- }
-
- private ProcedureLockState acquireLock(Procedure<Env> proc) {
- if (proc.hasLock()) {
- return ProcedureLockState.LOCK_ACQUIRED;
- }
- return proc.doAcquireLock(this.environment, store);
- }
-
- /**
- * do execute defined in procedure and then update store or remove completely in case it is a
- * child.
- *
- * @param procedure procedure
- * @return procedure lock state
- */
- private ProcedureLockState executeRollback(Procedure<Env> procedure) {
- try {
- procedure.doRollback(this.environment);
- } catch (IOException e) {
- LOG.error("Roll back failed for {}", procedure, e);
- } catch (InterruptedException e) {
- LOG.warn("Interrupted exception occured for {}", procedure, e);
- } catch (Throwable t) {
- LOG.error("CODE-BUG: runtime exception for {}", procedure, t);
- }
- cleanupAfterRollback(procedure);
- return ProcedureLockState.LOCK_ACQUIRED;
- }
-
- private void cleanupAfterRollback(Procedure<Env> procedure) {
- if (procedure.removeStackIndex()) {
- if (!procedure.isSuccess()) {
- procedure.setState(ProcedureState.ROLLEDBACK);
- }
-
- // update metrics on finishing the procedure (fail)
- procedure.updateMetricsOnFinish(getEnvironment(), procedure.elapsedTime(), false);
-
- if (procedure.hasParent()) {
- store.delete(procedure.getProcId());
- procedures.remove(procedure.getProcId());
- } else {
- final long[] childProcIds = rollbackStack.get(procedure.getProcId()).getSubprocedureIds();
- if (childProcIds != null) {
- store.delete(childProcIds);
- } else {
- store.update(procedure);
- }
- }
- } else {
- store.update(procedure);
- }
- }
-
- private void executeCompletionCleanup(Procedure<Env> proc) {
- if (proc.hasLock()) {
- releaseLock(proc, true);
- }
- }
-
- private void rootProcedureCleanup(Procedure<Env> proc) {
- executeCompletionCleanup(proc);
- CompletedProcedureContainer<Env> retainer = new CompletedProcedureContainer<>(proc);
- completed.put(proc.getProcId(), retainer);
- rollbackStack.remove(proc.getProcId());
- procedures.remove(proc.getProcId());
- }
-
- /**
- * Add a Procedure to executor.
- *
- * @param procedure procedure
- * @return procedure id
- */
- private long pushProcedure(Procedure<Env> procedure) {
- final long currentProcId = procedure.getProcId();
- // Update metrics on start of a procedure
- procedure.updateMetricsOnSubmit(getEnvironment());
- RootProcedureStack<Env> stack = new RootProcedureStack<>();
- rollbackStack.put(currentProcId, stack);
- procedures.put(currentProcId, procedure);
- scheduler.addBack(procedure);
- return procedure.getProcId();
- }
-
- private class WorkerThread extends StoppableThread {
- private final AtomicLong startTime = new AtomicLong(Long.MAX_VALUE);
- private final AtomicReference<Procedure<Env>> activeProcedure = new AtomicReference<>();
- protected long keepAliveTime = -1;
-
- public WorkerThread(ThreadGroup threadGroup) {
- this(threadGroup, "ProcedureCoreWorker-");
- }
-
- public WorkerThread(ThreadGroup threadGroup, String prefix) {
- super(threadGroup, prefix + workId.incrementAndGet());
- setDaemon(true);
- }
-
- @Override
- public void sendStopSignal() {
- scheduler.signalAll();
- }
-
- @Override
- public void run() {
- long lastUpdated = System.currentTimeMillis();
- try {
- while (isRunning() && keepAlive(lastUpdated)) {
- Procedure<Env> procedure = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);
- if (procedure == null) {
- Thread.sleep(1000);
- continue;
- }
- this.activeProcedure.set(procedure);
- activeExecutorCount.incrementAndGet();
- startTime.set(System.currentTimeMillis());
- executeProcedure(procedure);
- activeExecutorCount.decrementAndGet();
- LOG.trace(
- "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get());
- this.activeProcedure.set(null);
- lastUpdated = System.currentTimeMillis();
- startTime.set(lastUpdated);
- }
-
- } catch (Exception e) {
- if (this.activeProcedure.get() != null) {
- LOG.warn(
- "Exception happened when worker {} execute procedure {}",
- getName(),
- this.activeProcedure.get(),
- e);
- }
- } finally {
- LOG.info("Procedure worker {} terminated.", getName());
- }
- workerThreads.remove(this);
- }
-
- protected boolean keepAlive(long lastUpdated) {
- return true;
- }
-
- @Override
- public String toString() {
- Procedure<?> p = this.activeProcedure.get();
- return getName() + "(pid=" + (p == null ? NO_PROC_ID : p.getProcId() + ")");
- }
-
- /**
- * @return the time since the current procedure is running
- */
- public long getCurrentRunTime() {
- return System.currentTimeMillis() - startTime.get();
- }
- }
-
- // A temporary worker thread will be launched when too many core workers are stuck.
- // They will timeout after keepAliveTime if there is no procedure to run.
- private final class TemporaryWorkerThread extends WorkerThread {
-
- public TemporaryWorkerThread(ThreadGroup group) {
- super(group, "ProcedureTemporaryWorker-");
- this.keepAliveTime = TimeUnit.SECONDS.toMillis(10);
- }
-
- @Override
- protected boolean keepAlive(long lastUpdate) {
- return System.currentTimeMillis() - lastUpdate < keepAliveTime;
- }
- }
-
- private final class WorkerMonitor extends InternalProcedure<Env> {
- private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 30000; // 30sec
-
- private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 60000; // 60sec
-
- private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck
-
- public WorkerMonitor() {
- super(DEFAULT_WORKER_MONITOR_INTERVAL);
- updateTimestamp();
- }
-
- private int calculateRunningAndStuckWorkers() {
- // Check if any of the worker is stuck
- int runningCount = 0, stuckCount = 0;
- for (WorkerThread worker : workerThreads) {
- final Procedure<?> proc = worker.activeProcedure.get();
- if (proc == null) {
- continue;
- }
- runningCount++;
- // WARN the worker is stuck
- if (worker.getCurrentRunTime() > DEFAULT_WORKER_STUCK_THRESHOLD) {
- stuckCount++;
- LOG.warn(
- "Worker stuck {}({}), run time {} ms",
- worker,
- proc.getProcType(),
- worker.getCurrentRunTime());
- }
- LOG.info(
- "Procedure workers: {} is running, {} is running and stuck", runningCount, stuckCount);
- }
- return stuckCount;
- }
-
- private void checkThreadCount(final int stuckCount) {
- // Nothing to do if there are no runnable tasks
- if (stuckCount < 1 || !scheduler.hasRunnables()) {
- return;
- }
- // Add a new thread if the worker stuck percentage exceed the threshold limit
- // and every handler is active.
- final float stuckPerc = ((float) stuckCount) / workerThreads.size();
- // Let's add new worker thread more aggressively, as they will timeout finally if there is no
- // work to do.
- if (stuckPerc >= DEFAULT_WORKER_ADD_STUCK_PERCENTAGE && workerThreads.size() < maxPoolSize) {
- final TemporaryWorkerThread worker = new TemporaryWorkerThread(threadGroup);
- workerThreads.add(worker);
- worker.start();
- LOG.debug("Added new worker thread {}", worker);
- }
- }
-
- @Override
- protected void periodicExecute(Env env) {
- final int stuckCount = calculateRunningAndStuckWorkers();
- checkThreadCount(stuckCount);
- updateTimestamp();
- }
- }
-
- public int getWorkerThreadCount() {
- return workerThreads.size();
- }
-
- public long getActiveWorkerThreadCount() {
- return workerThreads.stream().filter(worker -> worker.activeProcedure.get() != null).count();
- }
-
- public boolean isRunning() {
- return running.get();
- }
-
- public void stop() {
- if (!running.getAndSet(false)) {
- return;
- }
- LOG.info("Stopping");
- scheduler.stop();
- timeoutExecutor.sendStopSignal();
- }
-
- public void join() {
- timeoutExecutor.awaitTermination();
- workerMonitorExecutor.awaitTermination();
- for (WorkerThread workerThread : workerThreads) {
- workerThread.awaitTermination();
- }
- try {
- threadGroup.destroy();
- } catch (IllegalThreadStateException e) {
- LOG.warn(
- "ProcedureExecutor threadGroup {} contains running threads which are used by non-procedure module.",
- this.threadGroup);
- this.threadGroup.list();
- }
- }
-
- public boolean isStarted(long procId) {
- Procedure<Env> procedure = procedures.get(procId);
- if (procedure == null) {
- return completed.get(procId) != null;
- }
- return procedure.wasExecuted();
- }
-
- public boolean isFinished(final long procId) {
- return !procedures.containsKey(procId);
- }
-
- public ConcurrentHashMap<Long, Procedure<Env>> getProcedures() {
- return procedures;
- }
-
- // -----------------------------CLIENT IMPLEMENTATION-----------------------------------
- /**
- * Submit a new root-procedure to the executor, called by client.
- *
- * @param procedure root procedure
- * @return procedure id
- */
- public long submitProcedure(Procedure<Env> procedure) {
- Preconditions.checkArgument(procedure.getState() == ProcedureState.INITIALIZING);
- Preconditions.checkArgument(!procedure.hasParent(), "Unexpected parent", procedure);
- // Initialize the procedure
- procedure.setProcId(store.getNextProcId());
- procedure.setProcRunnable();
- // Commit the transaction
- store.update(procedure);
- LOG.debug("{} is stored.", procedure);
- // Add the procedure to the executor
- return pushProcedure(procedure);
- }
-
- public ProcedureScheduler getScheduler() {
- return scheduler;
- }
-
- public Env getEnvironment() {
- return environment;
- }
-
- public IProcedureStore getStore() {
- return store;
- }
-
- public RootProcedureStack<Env> getRollbackStack(long rootProcId) {
- return rollbackStack.get(rootProcId);
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.commons.concurrent.ThreadName;
+1import org.apache.iotdb.commons.utils.TestOnly;
+1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+1import org.apache.iotdb.confignode.procedure.scheduler.SimpleProcedureScheduler;
+1import org.apache.iotdb.confignode.procedure.state.ProcedureLockState;
+1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+1import org.apache.iotdb.confignode.procedure.store.IProcedureStore;
+1
+1import com.google.common.base.Preconditions;
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.io.IOException;
+1import java.util.ArrayDeque;
+1import java.util.ArrayList;
+1import java.util.Arrays;
+1import java.util.Deque;
+1import java.util.HashSet;
+1import java.util.List;
+1import java.util.Objects;
+1import java.util.Set;
+1import java.util.concurrent.ConcurrentHashMap;
+1import java.util.concurrent.CopyOnWriteArrayList;
+1import java.util.concurrent.TimeUnit;
+1import java.util.concurrent.atomic.AtomicBoolean;
+1import java.util.concurrent.atomic.AtomicInteger;
+1import java.util.concurrent.atomic.AtomicLong;
+1import java.util.concurrent.atomic.AtomicReference;
+1
+1import static org.apache.iotdb.confignode.procedure.Procedure.NO_PROC_ID;
+1
+1public class ProcedureExecutor<Env> {
+1 private static final Logger LOG = LoggerFactory.getLogger(ProcedureExecutor.class);
+1
+1 private final ConcurrentHashMap<Long, CompletedProcedureContainer<Env>> completed =
+1 new ConcurrentHashMap<>();
+1
+1 private final ConcurrentHashMap<Long, RootProcedureStack<Env>> rollbackStack =
+1 new ConcurrentHashMap<>();
+1
+1 private final ConcurrentHashMap<Long, Procedure<Env>> procedures = new ConcurrentHashMap<>();
+1
+1 private ThreadGroup threadGroup;
+1
+1 private CopyOnWriteArrayList<WorkerThread> workerThreads;
+1
+1 private TimeoutExecutorThread<Env> timeoutExecutor;
+1
+1 private TimeoutExecutorThread<Env> workerMonitorExecutor;
+1
+1 private int corePoolSize;
+1 private int maxPoolSize;
+1
+1 private final ProcedureScheduler scheduler;
+1
+1 private final AtomicLong workId = new AtomicLong(0);
+1 private final AtomicInteger activeExecutorCount = new AtomicInteger(0);
+1 private final AtomicBoolean running = new AtomicBoolean(false);
+1 private final Env environment;
+1 private final IProcedureStore<Env> store;
+1
+1 public ProcedureExecutor(
+1 final Env environment, final IProcedureStore<Env> store, final ProcedureScheduler scheduler) {
+1 this.environment = environment;
+1 this.scheduler = scheduler;
+1 this.store = store;
+1 }
+1
+1 @TestOnly
+1 public ProcedureExecutor(final Env environment, final IProcedureStore<Env> store) {
+1 this(environment, store, new SimpleProcedureScheduler());
+1 }
+1
+1 public void init(int numThreads) {
+1 this.corePoolSize = numThreads;
+1 this.maxPoolSize = 10 * numThreads;
+1 this.threadGroup = new ThreadGroup(ThreadName.CONFIG_NODE_PROCEDURE_WORKER.getName());
+1 this.timeoutExecutor =
+1 new TimeoutExecutorThread<>(
+1 this, threadGroup, ThreadName.CONFIG_NODE_TIMEOUT_EXECUTOR.getName());
+1 this.workerMonitorExecutor =
+1 new TimeoutExecutorThread<>(
+1 this, threadGroup, ThreadName.CONFIG_NODE_WORKER_THREAD_MONITOR.getName());
+1 workId.set(0);
+1 workerThreads = new CopyOnWriteArrayList<>();
+1 for (int i = 0; i < corePoolSize; i++) {
+1 workerThreads.add(new WorkerThread(threadGroup));
+1 }
+1 // Add worker monitor
+1 workerMonitorExecutor.add(new WorkerMonitor());
+1
+1 scheduler.start();
+1 recover();
+1 }
+1
+1 private void recover() {
+1 // 1.Build rollback stack
+1 List<Procedure<Env>> procedureList = getProcedureListFromDifferentVersion();
+1 // Load procedure wal file
+1 for (Procedure<Env> proc : procedureList) {
+1 if (proc.isFinished()) {
+1 completed.putIfAbsent(proc.getProcId(), new CompletedProcedureContainer<>(proc));
+1 } else {
+1 if (!proc.hasParent()) {
+1 rollbackStack.put(proc.getProcId(), new RootProcedureStack<>());
+1 }
+1 procedures.putIfAbsent(proc.getProcId(), proc);
+1 }
+1 }
+1 List<Procedure<Env>> runnableList = new ArrayList<>();
+1 List<Procedure<Env>> failedList = new ArrayList<>();
+1 List<Procedure<Env>> waitingList = new ArrayList<>();
+1 List<Procedure<Env>> waitingTimeoutList = new ArrayList<>();
+1 for (Procedure<Env> proc : procedureList) {
+1 if (proc.isFinished() && !proc.hasParent()) {
+1 continue;
+1 }
+1 long rootProcedureId = getRootProcedureId(proc);
+1 if (proc.hasParent()) {
+1 Procedure<Env> parent = procedures.get(proc.getParentProcId());
+1 if (parent != null && !proc.isFinished()) {
+1 parent.incChildrenLatch();
+1 }
+1 }
+1 RootProcedureStack<Env> rootStack = rollbackStack.get(rootProcedureId);
+1 if (rootStack != null) {
+1 rootStack.loadStack(proc);
+1 }
+1 proc.setRootProcedureId(rootProcedureId);
+1 switch (proc.getState()) {
+1 case RUNNABLE:
+1 runnableList.add(proc);
+1 break;
+1 case FAILED:
+1 failedList.add(proc);
+1 break;
+1 case WAITING:
+1 waitingList.add(proc);
+1 break;
+1 case WAITING_TIMEOUT:
+1 waitingTimeoutList.add(proc);
+1 break;
+1 case ROLLEDBACK:
+1 case INITIALIZING:
+1 LOG.error("Unexpected state:{} for {}", proc.getState(), proc);
+1 throw new UnsupportedOperationException("Unexpected state");
+1 default:
+1 break;
+1 }
+1 }
+1
+1 waitingList.forEach(
+1 procedure -> {
+1 if (!procedure.hasChildren()) {
+1 // Normally, WAITING procedures should be wakened by its children.
+1 // But, there is a case that, all the children are successful, and before
+1 // they can wake up their parent procedure, the master was killed.
+1 // So, during recovering the procedures from ProcedureWal, its children
+1 // are not loaded because of their SUCCESS state.
+1 // So we need to continue to run this WAITING procedure. Before
+1 // executing, we need to set its state to RUNNABLE.
+1 procedure.setState(ProcedureState.RUNNABLE);
+1 runnableList.add(procedure);
+1 }
+1 });
+1 restoreLocks();
+1
+1 waitingTimeoutList.forEach(timeoutExecutor::add);
+1
+1 failedList.forEach(scheduler::addBack);
+1 runnableList.forEach(scheduler::addBack);
+1 scheduler.signalAll();
+1 }
+1
+1 private List<Procedure<Env>> getProcedureListFromDifferentVersion() {
+1 if (store.isOldVersionProcedureStore()) {
+1 LOG.info("Old procedure directory detected, upgrade beginning...");
+1 return store.load();
+1 } else {
+1 return store.getProcedures();
+1 }
+1 }
+1
+1 /**
+1 * Helper to look up the root Procedure ID.
+1 *
+1 * @param proc given a specified procedure.
+1 */
+1 Long getRootProcedureId(Procedure<Env> proc) {
+1 while (proc.hasParent()) {
+1 proc = procedures.get(proc.getParentProcId());
+1 if (proc == null) {
+1 return NO_PROC_ID;
+1 }
+1 }
+1 return proc.getProcId();
+1 }
+1
+1 private void releaseLock(Procedure<Env> procedure, boolean force) {
+1 if (force || !procedure.holdLock(this.environment) || procedure.isFinished()) {
+1 procedure.doReleaseLock(this.environment, store);
+1 }
+1 }
+1
+1 private void restoreLock(Procedure procedure, Set<Long> restored) {
+1 procedure.restoreLock(environment);
+1 restored.add(procedure.getProcId());
+1 }
+1
+1 private void restoreLocks(Deque<Procedure<Env>> stack, Set<Long> restored) {
+1 while (!stack.isEmpty()) {
+1 restoreLock(stack.pop(), restored);
+1 }
+1 }
+1
+1 private void restoreLocks() {
+1 Set<Long> restored = new HashSet<>();
+1 Deque<Procedure<Env>> stack = new ArrayDeque<>();
+1 procedures
+1 .values()
+1 .forEach(
+1 procedure -> {
+1 while (procedure != null) {
+1 if (restored.contains(procedure.getProcId())) {
+1 restoreLocks(stack, restored);
+1 return;
+1 }
+1 if (!procedure.hasParent()) {
+1 restoreLock(procedure, restored);
+1 restoreLocks(stack, restored);
+1 return;
+1 }
+1 stack.push(procedure);
+1 procedure = procedures.get(procedure.getParentProcId());
+1 }
+1 });
+1 }
+1
+1 public void startWorkers() {
+1 if (!running.compareAndSet(false, true)) {
+1 LOG.warn("Already running");
+1 return;
+1 }
+1 timeoutExecutor.start();
+1 workerMonitorExecutor.start();
+1 for (WorkerThread workerThread : workerThreads) {
+1 workerThread.start();
+1 }
+1 LOG.info("{} procedure workers are started.", workerThreads.size());
+1 }
+1
+1 public void startCompletedCleaner(long cleanTimeInterval, long cleanEvictTTL) {
+1 addInternalProcedure(
+1 new CompletedProcedureRecycler(store, completed, cleanTimeInterval, cleanEvictTTL));
+1 }
+1
+1 public void addInternalProcedure(InternalProcedure interalProcedure) {
+1 if (interalProcedure == null) {
+1 return;
+1 }
+1 interalProcedure.setState(ProcedureState.WAITING_TIMEOUT);
+1 timeoutExecutor.add(interalProcedure);
+1 }
+1
+1 public boolean removeInternalProcedure(InternalProcedure internalProcedure) {
+1 if (internalProcedure == null) {
+1 return true;
+1 }
+1 internalProcedure.setState(ProcedureState.SUCCESS);
+1 return timeoutExecutor.remove(internalProcedure);
+1 }
+1
+1 /**
+1 * Executes procedure
+1 *
+1 * <p>Calls doExecute() if success and return subprocedures submit sub procs set the state to
+1 * WAITING, wait for all sub procs completed. else if no sub procs procedure completed
+1 * successfully set procedure's parent to RUNNABLE in case of failure start rollback of the
+1 * procedure.
+1 *
+1 * @param proc procedure
+1 */
+1 private void executeProcedure(Procedure<Env> proc) {
+1 if (proc.isFinished()) {
+1 LOG.debug("{} is already finished.", proc);
+1 return;
+1 }
+1 final Long rootProcId = getRootProcedureId(proc);
+1 if (rootProcId == null) {
+1 LOG.warn("Rollback because parent is done/rolledback, proc is {}", proc);
+1 executeRollback(proc);
+1 return;
+1 }
+1 RootProcedureStack<Env> rootProcStack = rollbackStack.get(rootProcId);
+1 if (rootProcStack == null) {
+1 LOG.warn("Rollback stack is null for {}", proc.getProcId());
+1 return;
+1 }
+1 ProcedureLockState lockState = null;
+1 try {
+1 do {
+1 if (!rootProcStack.acquire()) {
+1 if (rootProcStack.setRollback()) {
+1 lockState = executeRootStackRollback(rootProcId, rootProcStack);
+1 switch (lockState) {
+1 case LOCK_ACQUIRED:
+1 break;
+1 case LOCK_EVENT_WAIT:
+1 LOG.info("LOCK_EVENT_WAIT rollback {}", proc);
+1 rootProcStack.unsetRollback();
+1 break;
+1 case LOCK_YIELD_WAIT:
+1 rootProcStack.unsetRollback();
+1 scheduler.yield(proc);
+1 break;
+1 default:
+1 throw new UnsupportedOperationException();
+1 }
+1 } else {
+1 if (!proc.wasExecuted()) {
+1 switch (executeRollback(proc)) {
+1 case LOCK_ACQUIRED:
+1 break;
+1 case LOCK_EVENT_WAIT:
+1 LOG.info("LOCK_EVENT_WAIT can't rollback child running for {}", proc);
+1 break;
+1 case LOCK_YIELD_WAIT:
+1 scheduler.yield(proc);
+1 break;
+1 default:
+1 throw new UnsupportedOperationException();
+1 }
+1 }
+1 }
+1 break;
+1 }
+1 lockState = acquireLock(proc);
+1 switch (lockState) {
+1 case LOCK_ACQUIRED:
+1 executeProcedure(rootProcStack, proc);
+1 break;
+1 case LOCK_YIELD_WAIT:
+1 case LOCK_EVENT_WAIT:
+1 LOG.info("{} lockstate is {}", proc, lockState);
+1 break;
+1 default:
+1 throw new UnsupportedOperationException();
+1 }
+1 rootProcStack.release();
+1
+1 if (proc.isSuccess()) {
+1 // update metrics on finishing the procedure
+1 proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), true);
+1 LOG.debug("{} finished in {}ms successfully.", proc, proc.elapsedTime());
+1 if (proc.getProcId() == rootProcId) {
+1 rootProcedureCleanup(proc);
+1 } else {
+1 executeCompletionCleanup(proc);
+1 }
+1 return;
+1 }
+1
+1 } while (rootProcStack.isFailed());
+1 } finally {
+1 // Only after procedure has completed execution can it be allowed to be rescheduled to prevent
+1 // data races
+1 if (Objects.equals(lockState, ProcedureLockState.LOCK_EVENT_WAIT)) {
+1 LOG.info("procedureId {} wait for lock.", proc.getProcId());
+1 ((ConfigNodeProcedureEnv) this.environment).getNodeLock().waitProcedure(proc);
+1 }
+1 }
+1 }
+1
+1 /**
+1 * execute procedure and submit its children
+1 *
+1 * @param rootProcStack procedure's root proc stack
+1 * @param proc procedure
+1 */
+1 private void executeProcedure(RootProcedureStack rootProcStack, Procedure<Env> proc) {
+1 if (proc.getState() != ProcedureState.RUNNABLE) {
+1 LOG.error(
+1 "The executing procedure should in RUNNABLE state, but it's not. Procedure is {}", proc);
+1 return;
+1 }
+1 boolean reExecute;
+1
+1 Procedure<Env>[] subprocs = null;
+1 do {
+1 reExecute = false;
+1 try {
+1 subprocs = proc.doExecute(this.environment);
+1 if (subprocs != null && subprocs.length == 0) {
+1 subprocs = null;
+1 }
+1 } catch (InterruptedException e) {
+1 LOG.warn("Interrupt during execution, suspend or retry it later.", e);
+1 yieldProcedure(proc);
+1 } catch (Throwable e) {
+1 LOG.error("CODE-BUG:{}", proc, e);
+1 proc.setFailure(new ProcedureException(e.getMessage(), e));
+1 }
+1
+1 if (!proc.isFailed()) {
+1 if (subprocs != null) {
+1 if (subprocs.length == 1 && subprocs[0] == proc) {
+1 subprocs = null;
+1 reExecute = true;
+1 } else {
+1 subprocs = initializeChildren(rootProcStack, proc, subprocs);
+1 LOG.info("Initialized sub procs:{}", Arrays.toString(subprocs));
+1 }
+1 } else if (proc.getState() == ProcedureState.WAITING_TIMEOUT) {
+1 LOG.info("Added into timeoutExecutor {}", proc);
+1 } else {
+1 proc.setState(ProcedureState.SUCCESS);
+1 }
+1 }
+1 // add procedure into rollback stack.
+1 rootProcStack.addRollbackStep(proc);
+1
+1 updateStoreOnExecution(rootProcStack, proc, subprocs);
+1
+1 if (!store.isRunning()) {
+1 return;
+1 }
+1
+1 if (proc.isRunnable() && proc.isYieldAfterExecution(this.environment)) {
+1 yieldProcedure(proc);
+1 return;
+1 }
+1 } while (reExecute);
+1
+1 if (subprocs != null && !proc.isFailed()) {
+1 submitChildrenProcedures(subprocs);
+1 }
+1
+1 releaseLock(proc, false);
+1 if (proc.isFinished() && proc.hasParent()) {
+1 countDownChildren(rootProcStack, proc);
+1 }
+1 }
+1
+1 /**
+1 * Serve as a countdown latch to check whether all children have already completed.
+1 *
+1 * @param rootProcStack root procedure stack
+1 * @param proc proc
+1 */
+1 private void countDownChildren(RootProcedureStack rootProcStack, Procedure<Env> proc) {
+1 Procedure<Env> parent = procedures.get(proc.getParentProcId());
+1 if (parent == null && rootProcStack.isRollingback()) {
+1 return;
+1 }
+1 if (parent != null && parent.tryRunnable()) {
+1 // If success, means all its children have completed, move parent to front of the queue.
+1 store.update(parent);
+1 scheduler.addFront(parent);
+1 LOG.info(
+1 "Finished subprocedure pid={}, resume processing ppid={}",
+1 proc.getProcId(),
+1 parent.getProcId());
+1 }
+1 }
+1
+1 /**
+1 * Submit children procedures.
+1 *
+1 * @param subprocs children procedures
+1 */
+1 private void submitChildrenProcedures(Procedure<Env>[] subprocs) {
+1 for (Procedure<Env> subproc : subprocs) {
+1 subproc.updateMetricsOnSubmit(getEnvironment());
+1 procedures.put(subproc.getProcId(), subproc);
+1 scheduler.addFront(subproc);
+1 LOG.info("Sub-Procedure pid={} has been submitted", subproc.getProcId());
+1 }
+1 }
+1
+1 private void updateStoreOnExecution(
+1 RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
+1 if (subprocs != null && !proc.isFailed()) {
+1 if (LOG.isDebugEnabled()) {
+1 LOG.debug("Stored {}, children {}", proc, Arrays.toString(subprocs));
+1 }
+1 store.update(subprocs);
+1 } else {
+1 LOG.debug("Store update {}", proc);
+1 if (proc.isFinished() && !proc.hasParent()) {
+1 final long[] childProcIds = rootProcStack.getSubprocedureIds();
+1 if (childProcIds != null) {
+1 store.delete(childProcIds);
+1 for (long childProcId : childProcIds) {
+1 procedures.remove(childProcId);
+1 }
+1 } else {
+1 store.update(proc);
+1 }
+1 } else {
+1 store.update(proc);
+1 }
+1 }
+1 }
+1
+1 private Procedure<Env>[] initializeChildren(
+1 RootProcedureStack rootProcStack, Procedure<Env> proc, Procedure<Env>[] subprocs) {
+1 final long rootProcedureId = getRootProcedureId(proc);
+1 for (int i = 0; i < subprocs.length; i++) {
+1 Procedure<Env> subproc = subprocs[i];
+1 if (subproc == null) {
+1 String errMsg = "subproc[" + i + "] is null, aborting procedure";
+1 proc.setFailure(new ProcedureException((errMsg), new IllegalArgumentException(errMsg)));
+1 return null;
+1 }
+1 subproc.setParentProcId(proc.getProcId());
+1 subproc.setRootProcId(rootProcedureId);
+1 subproc.setProcId(store.getNextProcId());
+1 subproc.setProcRunnable();
+1 rootProcStack.addSubProcedure(subproc);
+1 }
+1
+1 if (!proc.isFailed()) {
+1 proc.setChildrenLatch(subprocs.length);
+1 switch (proc.getState()) {
+1 case RUNNABLE:
+1 proc.setState(ProcedureState.WAITING);
+1 break;
+1 case WAITING_TIMEOUT:
+1 timeoutExecutor.add(proc);
+1 break;
+1 default:
+1 break;
+1 }
+1 }
+1 return subprocs;
+1 }
+1
+1 private void yieldProcedure(Procedure<Env> proc) {
+1 releaseLock(proc, false);
+1 scheduler.yield(proc);
+1 }
+1
+1 /**
+1 * Rollback full root procedure stack.
+1 *
+1 * @param rootProcId root procedure id
+1 * @param procedureStack root procedure stack
+1 * @return lock state
+1 */
+1 private ProcedureLockState executeRootStackRollback(
+1 Long rootProcId, RootProcedureStack procedureStack) {
+1 Procedure<Env> rootProcedure = procedures.get(rootProcId);
+1 ProcedureException exception = rootProcedure.getException();
+1 if (exception == null) {
+1 exception = procedureStack.getException();
+1 rootProcedure.setFailure(exception);
+1 store.update(rootProcedure);
+1 }
+1 List<Procedure<Env>> subprocStack = procedureStack.getSubproceduresStack();
+1 int stackTail = subprocStack.size();
+1 while (stackTail-- > 0) {
+1 Procedure<Env> procedure = subprocStack.get(stackTail);
+1 if (procedure.isSuccess()) {
+1 subprocStack.remove(stackTail);
+1 cleanupAfterRollback(procedure);
+1 continue;
+1 }
+1 ProcedureLockState lockState = acquireLock(procedure);
+1 if (lockState != ProcedureLockState.LOCK_ACQUIRED) {
+1 return lockState;
+1 }
+1 lockState = executeRollback(procedure);
+1 releaseLock(procedure, false);
+1
+1 boolean abortRollback = lockState != ProcedureLockState.LOCK_ACQUIRED;
+1 abortRollback |= !isRunning() || !store.isRunning();
+1 if (abortRollback) {
+1 return lockState;
+1 }
+1
+1 if (!procedure.isFinished() && procedure.isYieldAfterExecution(this.environment)) {
+1 return ProcedureLockState.LOCK_YIELD_WAIT;
+1 }
+1
+1 if (procedure != rootProcedure) {
+1 executeCompletionCleanup(procedure);
+1 }
+1 }
+1
+1 LOG.info("Rolled back {}, time duration is {}", rootProcedure, rootProcedure.elapsedTime());
+1 rootProcedureCleanup(rootProcedure);
+1 return ProcedureLockState.LOCK_ACQUIRED;
+1 }
+1
+1 private ProcedureLockState acquireLock(Procedure<Env> proc) {
+1 if (proc.hasLock()) {
+1 return ProcedureLockState.LOCK_ACQUIRED;
+1 }
+1 return proc.doAcquireLock(this.environment, store);
+1 }
+1
+1 /**
+1 * do execute defined in procedure and then update store or remove completely in case it is a
+1 * child.
+1 *
+1 * @param procedure procedure
+1 * @return procedure lock state
+1 */
+1 private ProcedureLockState executeRollback(Procedure<Env> procedure) {
+1 try {
+1 procedure.doRollback(this.environment);
+1 } catch (IOException e) {
+1 LOG.error("Roll back failed for {}", procedure, e);
+1 } catch (InterruptedException e) {
+1 LOG.warn("Interrupted exception occured for {}", procedure, e);
+1 } catch (Throwable t) {
+1 LOG.error("CODE-BUG: runtime exception for {}", procedure, t);
+1 }
+1 cleanupAfterRollback(procedure);
+1 return ProcedureLockState.LOCK_ACQUIRED;
+1 }
+1
+1 private void cleanupAfterRollback(Procedure<Env> procedure) {
+1 if (procedure.removeStackIndex()) {
+1 if (!procedure.isSuccess()) {
+1 procedure.setState(ProcedureState.ROLLEDBACK);
+1 }
+1
+1 // update metrics on finishing the procedure (fail)
+1 procedure.updateMetricsOnFinish(getEnvironment(), procedure.elapsedTime(), false);
+1
+1 if (procedure.hasParent()) {
+1 store.delete(procedure.getProcId());
+1 procedures.remove(procedure.getProcId());
+1 } else {
+1 final long[] childProcIds = rollbackStack.get(procedure.getProcId()).getSubprocedureIds();
+1 if (childProcIds != null) {
+1 store.delete(childProcIds);
+1 } else {
+1 store.update(procedure);
+1 }
+1 }
+1 } else {
+1 store.update(procedure);
+1 }
+1 }
+1
+1 private void executeCompletionCleanup(Procedure<Env> proc) {
+1 if (proc.hasLock()) {
+1 releaseLock(proc, true);
+1 }
+1 }
+1
+1 private void rootProcedureCleanup(Procedure<Env> proc) {
+1 executeCompletionCleanup(proc);
+1 CompletedProcedureContainer<Env> retainer = new CompletedProcedureContainer<>(proc);
+1 completed.put(proc.getProcId(), retainer);
+1 rollbackStack.remove(proc.getProcId());
+1 procedures.remove(proc.getProcId());
+1 }
+1
+1 /**
+1 * Add a Procedure to executor.
+1 *
+1 * @param procedure procedure
+1 * @return procedure id
+1 */
+1 private long pushProcedure(Procedure<Env> procedure) {
+1 final long currentProcId = procedure.getProcId();
+1 // Update metrics on start of a procedure
+1 procedure.updateMetricsOnSubmit(getEnvironment());
+1 RootProcedureStack<Env> stack = new RootProcedureStack<>();
+1 rollbackStack.put(currentProcId, stack);
+1 procedures.put(currentProcId, procedure);
+1 scheduler.addBack(procedure);
+1 return procedure.getProcId();
+1 }
+1
+1 private class WorkerThread extends StoppableThread {
+1 private final AtomicLong startTime = new AtomicLong(Long.MAX_VALUE);
+1 private final AtomicReference<Procedure<Env>> activeProcedure = new AtomicReference<>();
+1 protected long keepAliveTime = -1;
+1
+1 public WorkerThread(ThreadGroup threadGroup) {
+1 this(threadGroup, "ProcedureCoreWorker-");
+1 }
+1
+1 public WorkerThread(ThreadGroup threadGroup, String prefix) {
+1 super(threadGroup, prefix + workId.incrementAndGet());
+1 setDaemon(true);
+1 }
+1
+1 @Override
+1 public void sendStopSignal() {
+1 scheduler.signalAll();
+1 }
+1
+1 @Override
+1 public void run() {
+1 long lastUpdated = System.currentTimeMillis();
+1 try {
+1 while (isRunning() && keepAlive(lastUpdated)) {
+1 Procedure<Env> procedure = scheduler.poll(keepAliveTime, TimeUnit.MILLISECONDS);
+1 if (procedure == null) {
+1 Thread.sleep(1000);
+1 continue;
+1 }
+1 this.activeProcedure.set(procedure);
+1 activeExecutorCount.incrementAndGet();
+1 startTime.set(System.currentTimeMillis());
+1 executeProcedure(procedure);
+1 activeExecutorCount.decrementAndGet();
+1 LOG.trace(
+1 "Halt pid={}, activeCount={}", procedure.getProcId(), activeExecutorCount.get());
+1 this.activeProcedure.set(null);
+1 lastUpdated = System.currentTimeMillis();
+1 startTime.set(lastUpdated);
+1 }
+1
+1 } catch (Exception e) {
+1 if (this.activeProcedure.get() != null) {
+1 LOG.warn(
+1 "Exception happened when worker {} execute procedure {}",
+1 getName(),
+1 this.activeProcedure.get(),
+1 e);
+1 }
+1 } finally {
+1 LOG.info("Procedure worker {} terminated.", getName());
+1 }
+1 workerThreads.remove(this);
+1 }
+1
+1 protected boolean keepAlive(long lastUpdated) {
+1 return true;
+1 }
+1
+1 @Override
+1 public String toString() {
+1 Procedure<?> p = this.activeProcedure.get();
+1 return getName() + "(pid=" + (p == null ? NO_PROC_ID : p.getProcId() + ")");
+1 }
+1
+1 /**
+1 * @return the time since the current procedure is running
+1 */
+1 public long getCurrentRunTime() {
+1 return System.currentTimeMillis() - startTime.get();
+1 }
+1 }
+1
+1 // A temporary worker thread will be launched when too many core workers are stuck.
+1 // They will timeout after keepAliveTime if there is no procedure to run.
+1 private final class TemporaryWorkerThread extends WorkerThread {
+1
+1 public TemporaryWorkerThread(ThreadGroup group) {
+1 super(group, "ProcedureTemporaryWorker-");
+1 this.keepAliveTime = TimeUnit.SECONDS.toMillis(10);
+1 }
+1
+1 @Override
+1 protected boolean keepAlive(long lastUpdate) {
+1 return System.currentTimeMillis() - lastUpdate < keepAliveTime;
+1 }
+1 }
+1
+1 private final class WorkerMonitor extends InternalProcedure<Env> {
+1 private static final int DEFAULT_WORKER_MONITOR_INTERVAL = 30000; // 30sec
+1
+1 private static final int DEFAULT_WORKER_STUCK_THRESHOLD = 60000; // 60sec
+1
+1 private static final float DEFAULT_WORKER_ADD_STUCK_PERCENTAGE = 0.5f; // 50% stuck
+1
+1 public WorkerMonitor() {
+1 super(DEFAULT_WORKER_MONITOR_INTERVAL);
+1 updateTimestamp();
+1 }
+1
+1 private int calculateRunningAndStuckWorkers() {
+1 // Check if any of the worker is stuck
+1 int runningCount = 0, stuckCount = 0;
+1 for (WorkerThread worker : workerThreads) {
+1 final Procedure<?> proc = worker.activeProcedure.get();
+1 if (proc == null) {
+1 continue;
+1 }
+1 runningCount++;
+1 // WARN the worker is stuck
+1 if (worker.getCurrentRunTime() > DEFAULT_WORKER_STUCK_THRESHOLD) {
+1 stuckCount++;
+1 LOG.warn(
+1 "Worker stuck {}({}), run time {} ms",
+1 worker,
+1 proc.getProcType(),
+1 worker.getCurrentRunTime());
+1 }
+1 LOG.info(
+1 "Procedure workers: {} is running, {} is running and stuck", runningCount, stuckCount);
+1 }
+1 return stuckCount;
+1 }
+1
+1 private void checkThreadCount(final int stuckCount) {
+1 // Nothing to do if there are no runnable tasks
+1 if (stuckCount < 1 || !scheduler.hasRunnables()) {
+1 return;
+1 }
+1 // Add a new thread if the worker stuck percentage exceed the threshold limit
+1 // and every handler is active.
+1 final float stuckPerc = ((float) stuckCount) / workerThreads.size();
+1 // Let's add new worker thread more aggressively, as they will timeout finally if there is no
+1 // work to do.
+1 if (stuckPerc >= DEFAULT_WORKER_ADD_STUCK_PERCENTAGE && workerThreads.size() < maxPoolSize) {
+1 final TemporaryWorkerThread worker = new TemporaryWorkerThread(threadGroup);
+1 workerThreads.add(worker);
+1 worker.start();
+1 LOG.debug("Added new worker thread {}", worker);
+1 }
+1 }
+1
+1 @Override
+1 protected void periodicExecute(Env env) {
+1 final int stuckCount = calculateRunningAndStuckWorkers();
+1 checkThreadCount(stuckCount);
+1 updateTimestamp();
+1 }
+1 }
+1
+1 public int getWorkerThreadCount() {
+1 return workerThreads.size();
+1 }
+1
+1 public long getActiveWorkerThreadCount() {
+1 return workerThreads.stream().filter(worker -> worker.activeProcedure.get() != null).count();
+1 }
+1
+1 public boolean isRunning() {
+1 return running.get();
+1 }
+1
+1 public void stop() {
+1 if (!running.getAndSet(false)) {
+1 return;
+1 }
+1 LOG.info("Stopping");
+1 scheduler.stop();
+1 timeoutExecutor.sendStopSignal();
+1 }
+1
+1 public void join() {
+1 timeoutExecutor.awaitTermination();
+1 workerMonitorExecutor.awaitTermination();
+1 for (WorkerThread workerThread : workerThreads) {
+1 workerThread.awaitTermination();
+1 }
+1 try {
+1 threadGroup.destroy();
+1 } catch (IllegalThreadStateException e) {
+1 LOG.warn(
+1 "ProcedureExecutor threadGroup {} contains running threads which are used by non-procedure module.",
+1 this.threadGroup);
+1 this.threadGroup.list();
+1 }
+1 }
+1
+1 public boolean isStarted(long procId) {
+1 Procedure<Env> procedure = procedures.get(procId);
+1 if (procedure == null) {
+1 return completed.get(procId) != null;
+1 }
+1 return procedure.wasExecuted();
+1 }
+1
+1 public boolean isFinished(final long procId) {
+1 return !procedures.containsKey(procId);
+1 }
+1
+1 public ConcurrentHashMap<Long, Procedure<Env>> getProcedures() {
+1 return procedures;
+1 }
+1
+1 // -----------------------------CLIENT IMPLEMENTATION-----------------------------------
+1 /**
+1 * Submit a new root-procedure to the executor, called by client.
+1 *
+1 * @param procedure root procedure
+1 * @return procedure id
+1 */
+1 public long submitProcedure(Procedure<Env> procedure) {
+1 Preconditions.checkArgument(procedure.getState() == ProcedureState.INITIALIZING);
+1 Preconditions.checkArgument(!procedure.hasParent(), "Unexpected parent", procedure);
+1 // Initialize the procedure
+1 procedure.setProcId(store.getNextProcId());
+1 procedure.setProcRunnable();
+1 // Commit the transaction
+1 store.update(procedure);
+1 LOG.debug("{} is stored.", procedure);
+1 // Add the procedure to the executor
+1 return pushProcedure(procedure);
+1 }
+1
+1 public ProcedureScheduler getScheduler() {
+1 return scheduler;
+1 }
+1
+1 public Env getEnvironment() {
+1 return environment;
+1 }
+1
+1 public IProcedureStore getStore() {
+1 return store;
+1 }
+1
+1 public RootProcedureStack<Env> getRollbackStack(long rootProcId) {
+1 return rollbackStack.get(rootProcId);
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
index 25533667431..8ac6ff8f99b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/ProcedureMetrics.java
@@ -1,184 +1,185 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.commons.service.metric.enums.Metric;
-import org.apache.iotdb.commons.service.metric.enums.Tag;
-import org.apache.iotdb.confignode.manager.ProcedureManager;
-import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
-import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-import org.apache.iotdb.metrics.AbstractMetricService;
-import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
-import org.apache.iotdb.metrics.metricsets.IMetricSet;
-import org.apache.iotdb.metrics.type.Counter;
-import org.apache.iotdb.metrics.type.Timer;
-import org.apache.iotdb.metrics.utils.MetricLevel;
-import org.apache.iotdb.metrics.utils.MetricType;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class ProcedureMetrics implements IMetricSet {
- private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureMetrics.class);
-
- private final ProcedureManager procedureManager;
-
- private final ProcedureExecutor<ConfigNodeProcedureEnv> executor;
-
- private final ProcedureScheduler scheduler;
-
- private final Map<String, ProcedureMetricItems> metricItemsMap = new ConcurrentHashMap<>();
-
- private AbstractMetricService metricService;
-
- public ProcedureMetrics(ProcedureManager procedureManager) {
- this.procedureManager = procedureManager;
- this.executor = procedureManager.getExecutor();
- this.scheduler = procedureManager.getScheduler();
- }
-
- @Override
- public void bindTo(AbstractMetricService metricService) {
- this.metricService = metricService;
- bindThreadMetrics(metricService);
- }
-
- @Override
- public void unbindFrom(AbstractMetricService metricService) {
- unbindThreadMetrics(metricService);
-
- for (ProcedureMetricItems items : metricItemsMap.values()) {
- items.unbindMetricItems(metricService);
- }
- metricItemsMap.clear();
- }
-
- private void bindThreadMetrics(AbstractMetricService metricService) {
- metricService.createAutoGauge(
- Metric.PROCEDURE_WORKER_THREAD_COUNT.toString(),
- MetricLevel.CORE,
- executor,
- ProcedureExecutor::getWorkerThreadCount);
- metricService.createAutoGauge(
- Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString(),
- MetricLevel.CORE,
- executor,
- ProcedureExecutor::getActiveWorkerThreadCount);
- metricService.createAutoGauge(
- Metric.PROCEDURE_QUEUE_LENGTH.toString(),
- MetricLevel.CORE,
- scheduler,
- ProcedureScheduler::size);
- }
-
- private void unbindThreadMetrics(AbstractMetricService metricService) {
- metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_WORKER_THREAD_COUNT.toString());
- metricService.remove(
- MetricType.AUTO_GAUGE, Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString());
- metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_QUEUE_LENGTH.toString());
- }
-
- public void updateMetricsOnSubmit(String procType) {
- Optional.ofNullable(metricService)
- .ifPresent(
- service -> {
- metricItemsMap
- .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
- .updateMetricsOnSubmit();
- });
- }
-
- public void updateMetricsOnFinish(String procType, long runtime, boolean success) {
- Optional.ofNullable(metricService)
- .ifPresent(
- service -> {
- metricItemsMap
- .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
- .updateMetricsOnFinish(runtime, success);
- });
- }
-
- class ProcedureMetricItems {
- private final String procType;
- private Counter submittedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
- private Counter failedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
- private Timer executionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
-
- public ProcedureMetricItems(String procType, AbstractMetricService metricService) {
- this.procType = procType;
- bindMetricItems(metricService);
- }
-
- public void bindMetricItems(AbstractMetricService metricService) {
- this.submittedCounter =
- metricService.getOrCreateCounter(
- Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
- MetricLevel.CORE,
- Tag.TYPE.toString(),
- procType);
- this.failedCounter =
- metricService.getOrCreateCounter(
- Metric.PROCEDURE_FAILED_COUNT.toString(),
- MetricLevel.CORE,
- Tag.TYPE.toString(),
- procType);
- this.executionTimer =
- metricService.getOrCreateTimer(
- Metric.PROCEDURE_EXECUTION_TIME.toString(),
- MetricLevel.CORE,
- Tag.TYPE.toString(),
- procType);
- }
-
- public void unbindMetricItems(AbstractMetricService metricService) {
- metricService.remove(
- MetricType.COUNTER,
- Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
- Tag.TYPE.toString(),
- procType);
- metricService.remove(
- MetricType.COUNTER,
- Metric.PROCEDURE_FAILED_COUNT.toString(),
- Tag.TYPE.toString(),
- procType);
- metricService.remove(
- MetricType.TIMER,
- Metric.PROCEDURE_EXECUTION_TIME.toString(),
- Tag.TYPE.toString(),
- procType);
- }
-
- public void updateMetricsOnSubmit() {
- submittedCounter.inc();
- }
-
- public void updateMetricsOnFinish(long runtime, boolean success) {
- if (success) {
- executionTimer.updateMillis(runtime);
- } else {
- failedCounter.inc();
- }
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.commons.service.metric.enums.Metric;
+1import org.apache.iotdb.commons.service.metric.enums.Tag;
+1import org.apache.iotdb.confignode.manager.ProcedureManager;
+1import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+1import org.apache.iotdb.metrics.AbstractMetricService;
+1import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
+1import org.apache.iotdb.metrics.metricsets.IMetricSet;
+1import org.apache.iotdb.metrics.type.Counter;
+1import org.apache.iotdb.metrics.type.Timer;
+1import org.apache.iotdb.metrics.utils.MetricLevel;
+1import org.apache.iotdb.metrics.utils.MetricType;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.util.Map;
+1import java.util.Optional;
+1import java.util.concurrent.ConcurrentHashMap;
+1
+1public class ProcedureMetrics implements IMetricSet {
+1 private static final Logger LOGGER = LoggerFactory.getLogger(ProcedureMetrics.class);
+1
+1 private final ProcedureManager procedureManager;
+1
+1 private final ProcedureExecutor<ConfigNodeProcedureEnv> executor;
+1
+1 private final ProcedureScheduler scheduler;
+1
+1 private final Map<String, ProcedureMetricItems> metricItemsMap = new ConcurrentHashMap<>();
+1
+1 private AbstractMetricService metricService;
+1
+1 public ProcedureMetrics(ProcedureManager procedureManager) {
+1 this.procedureManager = procedureManager;
+1 this.executor = procedureManager.getExecutor();
+1 this.scheduler = procedureManager.getScheduler();
+1 }
+1
+1 @Override
+1 public void bindTo(AbstractMetricService metricService) {
+1 this.metricService = metricService;
+1 bindThreadMetrics(metricService);
+1 }
+1
+1 @Override
+1 public void unbindFrom(AbstractMetricService metricService) {
+1 unbindThreadMetrics(metricService);
+1
+1 for (ProcedureMetricItems items : metricItemsMap.values()) {
+1 items.unbindMetricItems(metricService);
+1 }
+1 metricItemsMap.clear();
+1 }
+1
+1 private void bindThreadMetrics(AbstractMetricService metricService) {
+1 metricService.createAutoGauge(
+1 Metric.PROCEDURE_WORKER_THREAD_COUNT.toString(),
+1 MetricLevel.CORE,
+1 executor,
+1 ProcedureExecutor::getWorkerThreadCount);
+1 metricService.createAutoGauge(
+1 Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString(),
+1 MetricLevel.CORE,
+1 executor,
+1 ProcedureExecutor::getActiveWorkerThreadCount);
+1 metricService.createAutoGauge(
+1 Metric.PROCEDURE_QUEUE_LENGTH.toString(),
+1 MetricLevel.CORE,
+1 scheduler,
+1 ProcedureScheduler::size);
+1 }
+1
+1 private void unbindThreadMetrics(AbstractMetricService metricService) {
+1 metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_WORKER_THREAD_COUNT.toString());
+1 metricService.remove(
+1 MetricType.AUTO_GAUGE, Metric.PROCEDURE_ACTIVE_WORKER_THREAD_COUNT.toString());
+1 metricService.remove(MetricType.AUTO_GAUGE, Metric.PROCEDURE_QUEUE_LENGTH.toString());
+1 }
+1
+1 public void updateMetricsOnSubmit(String procType) {
+1 Optional.ofNullable(metricService)
+1 .ifPresent(
+1 service -> {
+1 metricItemsMap
+1 .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
+1 .updateMetricsOnSubmit();
+1 });
+1 }
+1
+1 public void updateMetricsOnFinish(String procType, long runtime, boolean success) {
+1 Optional.ofNullable(metricService)
+1 .ifPresent(
+1 service -> {
+1 metricItemsMap
+1 .computeIfAbsent(procType, k -> new ProcedureMetricItems(k, service))
+1 .updateMetricsOnFinish(runtime, success);
+1 });
+1 }
+1
+1 class ProcedureMetricItems {
+1 private final String procType;
+1 private Counter submittedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+1 private Counter failedCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+1 private Timer executionTimer = DoNothingMetricManager.DO_NOTHING_TIMER;
+1
+1 public ProcedureMetricItems(String procType, AbstractMetricService metricService) {
+1 this.procType = procType;
+1 bindMetricItems(metricService);
+1 }
+1
+1 public void bindMetricItems(AbstractMetricService metricService) {
+1 this.submittedCounter =
+1 metricService.getOrCreateCounter(
+1 Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
+1 MetricLevel.CORE,
+1 Tag.TYPE.toString(),
+1 procType);
+1 this.failedCounter =
+1 metricService.getOrCreateCounter(
+1 Metric.PROCEDURE_FAILED_COUNT.toString(),
+1 MetricLevel.CORE,
+1 Tag.TYPE.toString(),
+1 procType);
+1 this.executionTimer =
+1 metricService.getOrCreateTimer(
+1 Metric.PROCEDURE_EXECUTION_TIME.toString(),
+1 MetricLevel.CORE,
+1 Tag.TYPE.toString(),
+1 procType);
+1 }
+1
+1 public void unbindMetricItems(AbstractMetricService metricService) {
+1 metricService.remove(
+1 MetricType.COUNTER,
+1 Metric.PROCEDURE_SUBMITTED_COUNT.toString(),
+1 Tag.TYPE.toString(),
+1 procType);
+1 metricService.remove(
+1 MetricType.COUNTER,
+1 Metric.PROCEDURE_FAILED_COUNT.toString(),
+1 Tag.TYPE.toString(),
+1 procType);
+1 metricService.remove(
+1 MetricType.TIMER,
+1 Metric.PROCEDURE_EXECUTION_TIME.toString(),
+1 Tag.TYPE.toString(),
+1 procType);
+1 }
+1
+1 public void updateMetricsOnSubmit() {
+1 submittedCounter.inc();
+1 }
+1
+1 public void updateMetricsOnFinish(long runtime, boolean success) {
+1 if (success) {
+1 executionTimer.updateMillis(runtime);
+1 } else {
+1 failedCounter.inc();
+1 }
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
index ff19872a550..88caab664cd 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/RootProcedureStack.java
@@ -1,177 +1,178 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.state.ProcedureState;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public class RootProcedureStack<Env> {
- private static final Logger LOG = LoggerFactory.getLogger(RootProcedureStack.class);
-
- private enum State {
- RUNNING, // The Procedure is running or ready to run
- FAILED, // The Procedure failed, waiting for the rollback executing
- ROLLINGBACK, // The Procedure failed and the execution was rolledback
- }
-
- private Set<Procedure<Env>> subprocs = null;
- private ArrayList<Procedure<Env>> subprocStack = null;
- private State state = State.RUNNING;
- private int running = 0;
-
- public synchronized boolean isFailed() {
- switch (state) {
- case ROLLINGBACK:
- case FAILED:
- return true;
- default:
- break;
- }
- return false;
- }
-
- public synchronized boolean isRollingback() {
- return state == State.ROLLINGBACK;
- }
-
- /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
- protected synchronized boolean setRollback() {
- if (running == 0 && state == State.FAILED) {
- state = State.ROLLINGBACK;
- return true;
- }
- return false;
- }
-
- /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
- protected synchronized void unsetRollback() {
- assert state == State.ROLLINGBACK;
- state = State.FAILED;
- }
-
- protected synchronized long[] getSubprocedureIds() {
- if (subprocs == null) {
- return new long[0];
- }
- return subprocs.stream().mapToLong(Procedure::getProcId).toArray();
- }
-
- protected synchronized List<Procedure<Env>> getSubproceduresStack() {
- return subprocStack;
- }
-
- protected synchronized ProcedureException getException() {
- if (subprocStack != null) {
- for (Procedure<Env> proc : subprocStack) {
- if (proc.hasException()) {
- return proc.getException();
- }
- }
- }
- return null;
- }
-
- /** Called by the {@link ProcedureExecutor} to mark the procedure step as running. */
- protected synchronized boolean acquire() {
- if (state != State.RUNNING) {
- return false;
- }
-
- running++;
- return true;
- }
-
- /** Called by the {@link ProcedureExecutor} to mark the procedure step as finished. */
- protected synchronized void release() {
- running--;
- }
-
- protected synchronized void abort() {
- if (state == State.RUNNING) {
- state = State.FAILED;
- }
- }
-
- /**
- * Called by the {@link ProcedureExecutor} after the procedure step is completed, to add the step
- * to the rollback list (or procedure stack).
- */
- protected synchronized void addRollbackStep(Procedure<Env> proc) {
- if (proc.isFailed()) {
- state = State.FAILED;
- }
- if (subprocStack == null) {
- subprocStack = new ArrayList<>();
- }
- proc.addStackIndex(subprocStack.size());
- LOG.trace("Add procedure {} as the {}th rollback step", proc, subprocStack.size());
- subprocStack.add(proc);
- }
-
- protected synchronized void addSubProcedure(Procedure<Env> proc) {
- if (!proc.hasParent()) {
- return;
- }
- if (subprocs == null) {
- subprocs = new HashSet<>();
- }
- subprocs.add(proc);
- }
-
- /**
- * Called on store load by the {@link ProcedureExecutor} to load part of the stack.
- *
- * <p>Each procedure has its own stack-positions. Which means we have to write to the store only
- * the Procedure we executed, and nothing else. On load we recreate the full stack by aggregating
- * each procedure stack-positions.
- */
- protected synchronized void loadStack(Procedure<Env> proc) {
- addSubProcedure(proc);
- int[] stackIndexes = proc.getStackIndexes();
- if (stackIndexes != null) {
- if (subprocStack == null) {
- subprocStack = new ArrayList<>();
- }
- int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();
- if (diff > 0) {
- subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
- while (diff-- > 0) {
- subprocStack.add(null);
- }
- }
- for (int stackIndex : stackIndexes) {
- subprocStack.set(stackIndex, proc);
- }
- }
- if (proc.getState() == ProcedureState.ROLLEDBACK) {
- state = State.ROLLINGBACK;
- } else if (proc.isFailed()) {
- state = State.FAILED;
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+1import org.apache.iotdb.confignode.procedure.state.ProcedureState;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.util.ArrayList;
+1import java.util.HashSet;
+1import java.util.List;
+1import java.util.Set;
+1
+1public class RootProcedureStack<Env> {
+1 private static final Logger LOG = LoggerFactory.getLogger(RootProcedureStack.class);
+1
+1 private enum State {
+1 RUNNING, // The Procedure is running or ready to run
+1 FAILED, // The Procedure failed, waiting for the rollback executing
+1 ROLLINGBACK, // The Procedure failed and the execution was rolledback
+1 }
+1
+1 private Set<Procedure<Env>> subprocs = null;
+1 private ArrayList<Procedure<Env>> subprocStack = null;
+1 private State state = State.RUNNING;
+1 private int running = 0;
+1
+1 public synchronized boolean isFailed() {
+1 switch (state) {
+1 case ROLLINGBACK:
+1 case FAILED:
+1 return true;
+1 default:
+1 break;
+1 }
+1 return false;
+1 }
+1
+1 public synchronized boolean isRollingback() {
+1 return state == State.ROLLINGBACK;
+1 }
+1
+1 /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
+1 protected synchronized boolean setRollback() {
+1 if (running == 0 && state == State.FAILED) {
+1 state = State.ROLLINGBACK;
+1 return true;
+1 }
+1 return false;
+1 }
+1
+1 /** Called by the {@link ProcedureExecutor} to mark rollback execution. */
+1 protected synchronized void unsetRollback() {
+1 assert state == State.ROLLINGBACK;
+1 state = State.FAILED;
+1 }
+1
+1 protected synchronized long[] getSubprocedureIds() {
+1 if (subprocs == null) {
+1 return new long[0];
+1 }
+1 return subprocs.stream().mapToLong(Procedure::getProcId).toArray();
+1 }
+1
+1 protected synchronized List<Procedure<Env>> getSubproceduresStack() {
+1 return subprocStack;
+1 }
+1
+1 protected synchronized ProcedureException getException() {
+1 if (subprocStack != null) {
+1 for (Procedure<Env> proc : subprocStack) {
+1 if (proc.hasException()) {
+1 return proc.getException();
+1 }
+1 }
+1 }
+1 return null;
+1 }
+1
+1 /** Called by the {@link ProcedureExecutor} to mark the procedure step as running. */
+1 protected synchronized boolean acquire() {
+1 if (state != State.RUNNING) {
+1 return false;
+1 }
+1
+1 running++;
+1 return true;
+1 }
+1
+1 /** Called by the {@link ProcedureExecutor} to mark the procedure step as finished. */
+1 protected synchronized void release() {
+1 running--;
+1 }
+1
+1 protected synchronized void abort() {
+1 if (state == State.RUNNING) {
+1 state = State.FAILED;
+1 }
+1 }
+1
+1 /**
+1 * Called by the {@link ProcedureExecutor} after the procedure step is completed, to add the step
+1 * to the rollback list (or procedure stack).
+1 */
+1 protected synchronized void addRollbackStep(Procedure<Env> proc) {
+1 if (proc.isFailed()) {
+1 state = State.FAILED;
+1 }
+1 if (subprocStack == null) {
+1 subprocStack = new ArrayList<>();
+1 }
+1 proc.addStackIndex(subprocStack.size());
+1 LOG.trace("Add procedure {} as the {}th rollback step", proc, subprocStack.size());
+1 subprocStack.add(proc);
+1 }
+1
+1 protected synchronized void addSubProcedure(Procedure<Env> proc) {
+1 if (!proc.hasParent()) {
+1 return;
+1 }
+1 if (subprocs == null) {
+1 subprocs = new HashSet<>();
+1 }
+1 subprocs.add(proc);
+1 }
+1
+1 /**
+1 * Called on store load by the {@link ProcedureExecutor} to load part of the stack.
+1 *
+1 * <p>Each procedure has its own stack-positions. Which means we have to write to the store only
+1 * the Procedure we executed, and nothing else. On load we recreate the full stack by aggregating
+1 * each procedure stack-positions.
+1 */
+1 protected synchronized void loadStack(Procedure<Env> proc) {
+1 addSubProcedure(proc);
+1 int[] stackIndexes = proc.getStackIndexes();
+1 if (stackIndexes != null) {
+1 if (subprocStack == null) {
+1 subprocStack = new ArrayList<>();
+1 }
+1 int diff = (1 + stackIndexes[stackIndexes.length - 1]) - subprocStack.size();
+1 if (diff > 0) {
+1 subprocStack.ensureCapacity(1 + stackIndexes[stackIndexes.length - 1]);
+1 while (diff-- > 0) {
+1 subprocStack.add(null);
+1 }
+1 }
+1 for (int stackIndex : stackIndexes) {
+1 subprocStack.set(stackIndex, proc);
+1 }
+1 }
+1 if (proc.getState() == ProcedureState.ROLLEDBACK) {
+1 state = State.ROLLINGBACK;
+1 } else if (proc.isFailed()) {
+1 state = State.FAILED;
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
index 2e5bf8dd203..5e6297d3d97 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/StoppableThread.java
@@ -1,55 +1,56 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class StoppableThread extends Thread {
-
- private static final int JOIN_TIMEOUT = 250;
- private static final Logger LOG = LoggerFactory.getLogger(StoppableThread.class);
-
- protected StoppableThread(ThreadGroup threadGroup, String name) {
- super(threadGroup, name);
- }
-
- public abstract void sendStopSignal();
-
- public void awaitTermination() {
- try {
- for (int i = 0; isAlive(); i++) {
- sendStopSignal();
- join(JOIN_TIMEOUT);
- if (i > 0 && (i % 8) == 0) {
- interrupt();
- }
- }
- } catch (InterruptedException e) {
- LOG.warn("{} join wait got interrupted", getName(), e);
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void run() {
- // empty method
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1public abstract class StoppableThread extends Thread {
+1
+1 private static final int JOIN_TIMEOUT = 250;
+1 private static final Logger LOG = LoggerFactory.getLogger(StoppableThread.class);
+1
+1 protected StoppableThread(ThreadGroup threadGroup, String name) {
+1 super(threadGroup, name);
+1 }
+1
+1 public abstract void sendStopSignal();
+1
+1 public void awaitTermination() {
+1 try {
+1 for (int i = 0; isAlive(); i++) {
+1 sendStopSignal();
+1 join(JOIN_TIMEOUT);
+1 if (i > 0 && (i % 8) == 0) {
+1 interrupt();
+1 }
+1 }
+1 } catch (InterruptedException e) {
+1 LOG.warn("{} join wait got interrupted", getName(), e);
+1 Thread.currentThread().interrupt();
+1 }
+1 }
+1
+1 @Override
+1 public void run() {
+1 // empty method
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
index 5aaf9a623f5..1b028fa1288 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/TimeoutExecutorThread.java
@@ -1,107 +1,108 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure;
-
-import java.util.concurrent.DelayQueue;
-import java.util.concurrent.Delayed;
-import java.util.concurrent.TimeUnit;
-
-public class TimeoutExecutorThread<Env> extends StoppableThread {
-
- private static final int DELAY_QUEUE_TIMEOUT = 20;
- private final ProcedureExecutor<Env> executor;
- private final DelayQueue<ProcedureDelayContainer<Env>> queue = new DelayQueue<>();
-
- public TimeoutExecutorThread(
- ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, String name) {
- super(threadGroup, name);
- setDaemon(true);
- this.executor = envProcedureExecutor;
- }
-
- public void add(Procedure<Env> procedure) {
- queue.add(new ProcedureDelayContainer<>(procedure));
- }
-
- public boolean remove(Procedure<Env> procedure) {
- return queue.remove(new ProcedureDelayContainer<>(procedure));
- }
-
- private ProcedureDelayContainer<Env> takeQuietly() {
- try {
- return queue.poll(DELAY_QUEUE_TIMEOUT, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- currentThread().interrupt();
- return null;
- }
- }
-
- @Override
- public void run() {
- while (executor.isRunning()) {
- ProcedureDelayContainer<Env> delayTask = takeQuietly();
- if (delayTask == null) {
- continue;
- }
- Procedure<Env> procedure = delayTask.getProcedure();
- if (procedure instanceof InternalProcedure) {
- InternalProcedure internal = (InternalProcedure) procedure;
- internal.periodicExecute(executor.getEnvironment());
- procedure.updateTimestamp();
- queue.add(delayTask);
- } else {
- if (procedure.setTimeoutFailure(executor.getEnvironment())) {
- long rootProcId = executor.getRootProcedureId(procedure);
- RootProcedureStack<Env> rollbackStack = executor.getRollbackStack(rootProcId);
- rollbackStack.abort();
- executor.getStore().update(procedure);
- executor.getScheduler().addFront(procedure);
- }
- }
- }
- }
-
- public void sendStopSignal() {}
-
- private static class ProcedureDelayContainer<Env> implements Delayed {
-
- private final Procedure<Env> procedure;
-
- public ProcedureDelayContainer(Procedure<Env> procedure) {
- this.procedure = procedure;
- }
-
- public Procedure<Env> getProcedure() {
- return procedure;
- }
-
- @Override
- public long getDelay(TimeUnit unit) {
- long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis();
- return unit.convert(delay, TimeUnit.MILLISECONDS);
- }
-
- @Override
- public int compareTo(Delayed other) {
- return Long.compare(
- this.getDelay(TimeUnit.MILLISECONDS), other.getDelay(TimeUnit.MILLISECONDS));
- }
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure;
+1
+1import java.util.concurrent.DelayQueue;
+1import java.util.concurrent.Delayed;
+1import java.util.concurrent.TimeUnit;
+1
+1public class TimeoutExecutorThread<Env> extends StoppableThread {
+1
+1 private static final int DELAY_QUEUE_TIMEOUT = 20;
+1 private final ProcedureExecutor<Env> executor;
+1 private final DelayQueue<ProcedureDelayContainer<Env>> queue = new DelayQueue<>();
+1
+1 public TimeoutExecutorThread(
+1 ProcedureExecutor<Env> envProcedureExecutor, ThreadGroup threadGroup, String name) {
+1 super(threadGroup, name);
+1 setDaemon(true);
+1 this.executor = envProcedureExecutor;
+1 }
+1
+1 public void add(Procedure<Env> procedure) {
+1 queue.add(new ProcedureDelayContainer<>(procedure));
+1 }
+1
+1 public boolean remove(Procedure<Env> procedure) {
+1 return queue.remove(new ProcedureDelayContainer<>(procedure));
+1 }
+1
+1 private ProcedureDelayContainer<Env> takeQuietly() {
+1 try {
+1 return queue.poll(DELAY_QUEUE_TIMEOUT, TimeUnit.SECONDS);
+1 } catch (InterruptedException e) {
+1 currentThread().interrupt();
+1 return null;
+1 }
+1 }
+1
+1 @Override
+1 public void run() {
+1 while (executor.isRunning()) {
+1 ProcedureDelayContainer<Env> delayTask = takeQuietly();
+1 if (delayTask == null) {
+1 continue;
+1 }
+1 Procedure<Env> procedure = delayTask.getProcedure();
+1 if (procedure instanceof InternalProcedure) {
+1 InternalProcedure internal = (InternalProcedure) procedure;
+1 internal.periodicExecute(executor.getEnvironment());
+1 procedure.updateTimestamp();
+1 queue.add(delayTask);
+1 } else {
+1 if (procedure.setTimeoutFailure(executor.getEnvironment())) {
+1 long rootProcId = executor.getRootProcedureId(procedure);
+1 RootProcedureStack<Env> rollbackStack = executor.getRollbackStack(rootProcId);
+1 rollbackStack.abort();
+1 executor.getStore().update(procedure);
+1 executor.getScheduler().addFront(procedure);
+1 }
+1 }
+1 }
+1 }
+1
+1 public void sendStopSignal() {}
+1
+1 private static class ProcedureDelayContainer<Env> implements Delayed {
+1
+1 private final Procedure<Env> procedure;
+1
+1 public ProcedureDelayContainer(Procedure<Env> procedure) {
+1 this.procedure = procedure;
+1 }
+1
+1 public Procedure<Env> getProcedure() {
+1 return procedure;
+1 }
+1
+1 @Override
+1 public long getDelay(TimeUnit unit) {
+1 long delay = procedure.getTimeoutTimestamp() - System.currentTimeMillis();
+1 return unit.convert(delay, TimeUnit.MILLISECONDS);
+1 }
+1
+1 @Override
+1 public int compareTo(Delayed other) {
+1 return Long.compare(
+1 this.getDelay(TimeUnit.MILLISECONDS), other.getDelay(TimeUnit.MILLISECONDS));
+1 }
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index 960d0a7977f..f234032107b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -1,894 +1,895 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.confignode.procedure.env;
-
-import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
-import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
-import org.apache.iotdb.commons.pipe.config.PipeConfig;
-import org.apache.iotdb.commons.trigger.TriggerInformation;
-import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
-import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
-import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
-import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
-import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
-import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
-import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
-import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
-import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
-import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan;
-import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
-import org.apache.iotdb.confignode.exception.AddConsensusGroupException;
-import org.apache.iotdb.confignode.exception.AddPeerException;
-import org.apache.iotdb.confignode.manager.ConfigManager;
-import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
-import org.apache.iotdb.confignode.manager.load.LoadManager;
-import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
-import org.apache.iotdb.confignode.manager.node.NodeManager;
-import org.apache.iotdb.confignode.manager.partition.PartitionManager;
-import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
-import org.apache.iotdb.confignode.persistence.partition.PartitionInfo;
-import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.confignode.procedure.scheduler.LockQueue;
-import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
-import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
-import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
-import org.apache.iotdb.consensus.exception.ConsensusException;
-import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
-import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
-import org.apache.iotdb.mpp.rpc.thrift.TCreatePipePluginInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
-import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq;
-import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
-import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
-import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushMultiTopicMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp;
-import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
-import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import org.apache.thrift.TException;
-import org.apache.tsfile.utils.Binary;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
-
-public class ConfigNodeProcedureEnv {
-
- private static final Logger LOG = LoggerFactory.getLogger(ConfigNodeProcedureEnv.class);
-
- /** Add or remove node lock. */
- private final LockQueue nodeLock = new LockQueue();
-
- private final ReentrantLock schedulerLock = new ReentrantLock(true);
-
- private final ReentrantLock submitRegionMigrateLock = new ReentrantLock(true);
-
- private final ConfigManager configManager;
-
- private final ProcedureScheduler scheduler;
-
- private final RegionMaintainHandler regionMaintainHandler;
-
- private final RemoveDataNodeHandler removeDataNodeHandler;
-
- private final ReentrantLock removeConfigNodeLock;
-
- public ConfigNodeProcedureEnv(ConfigManager configManager, ProcedureScheduler scheduler) {
- this.configManager = configManager;
- this.scheduler = scheduler;
- this.regionMaintainHandler = new RegionMaintainHandler(configManager);
- this.removeDataNodeHandler = new RemoveDataNodeHandler(configManager);
- this.removeConfigNodeLock = new ReentrantLock();
- }
-
- public ConfigManager getConfigManager() {
- return configManager;
- }
-
- /**
- * Delete ConfigNode cache, includes {@link ClusterSchemaInfo} and {@link PartitionInfo}.
- *
- * @param name database name
- * @param isGeneratedByPipe whether the deletion is triggered by pipe request
- * @return tsStatus
- */
- public TSStatus deleteDatabaseConfig(final String name, final boolean isGeneratedByPipe) {
- return getClusterSchemaManager()
- .deleteDatabase(new DeleteDatabasePlan(name), isGeneratedByPipe);
- }
-
- /**
- * Pre delete a database.
- *
- * @param preDeleteType execute/rollback
- * @param deleteSgName database name
- */
- public void preDeleteDatabase(
- final PreDeleteDatabasePlan.PreDeleteType preDeleteType, final String deleteSgName) {
- getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType);
- }
-
- /**
- * @param databaseName database name
- * @return ALL SUCCESS OR NOT
- * @throws IOException IOE
- * @throws TException Thrift IOE
- */
- public boolean invalidateCache(final String databaseName) throws IOException, TException {
- final List<TDataNodeConfiguration> allDataNodes = getNodeManager().getRegisteredDataNodes();
- final TInvalidateCacheReq invalidateCacheReq = new TInvalidateCacheReq();
- invalidateCacheReq.setStorageGroup(true);
- invalidateCacheReq.setFullPath(databaseName);
- for (final TDataNodeConfiguration dataNodeConfiguration : allDataNodes) {
- final int dataNodeId = dataNodeConfiguration.getLocation().getDataNodeId();
-
- // If the node is not alive, retry for up to 10 times
- NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
- int retryNum = 10;
- if (nodeStatus == NodeStatus.Unknown) {
- for (int i = 0; i < retryNum && nodeStatus == NodeStatus.Unknown; i++) {
- try {
- TimeUnit.MILLISECONDS.sleep(500);
- } catch (final InterruptedException e) {
- LOG.error("Sleep failed in ConfigNodeProcedureEnv: ", e);
- Thread.currentThread().interrupt();
- break;
- }
- nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
- }
- }
-
- if (nodeStatus == NodeStatus.Unknown) {
- LOG.warn(
- "Invalidate cache failed, because DataNode {} is Unknown",
- dataNodeConfiguration.getLocation().getInternalEndPoint());
- return false;
- }
-
- // Always invalidate PartitionCache first
- final TSStatus invalidatePartitionStatus =
- (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- dataNodeConfiguration.getLocation().getInternalEndPoint(),
- invalidateCacheReq,
- CnToDnSyncRequestType.INVALIDATE_PARTITION_CACHE);
-
- final TSStatus invalidateSchemaStatus =
- (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- dataNodeConfiguration.getLocation().getInternalEndPoint(),
- invalidateCacheReq,
- CnToDnSyncRequestType.INVALIDATE_SCHEMA_CACHE);
-
- if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) {
- LOG.error(
- "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine cache status is {}",
- invalidatePartitionStatus,
- invalidateSchemaStatus);
- return false;
- }
- }
- return true;
- }
-
- public boolean verifySucceed(TSStatus... status) {
- return Arrays.stream(status)
- .allMatch(tsStatus -> tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode());
- }
-
- /**
- * Let the remotely new ConfigNode build the ConsensusGroup.
- *
- * <p>Actually, the parameter of this method can be empty, adding new raft peer to exist group
- * should invoke createPeer(groupId, emptyList).
- *
- * @param tConfigNodeLocation New ConfigNode's location
- */
- public void addConsensusGroup(TConfigNodeLocation tConfigNodeLocation)
- throws AddConsensusGroupException {
- List<TConfigNodeLocation> configNodeLocations =
- new ArrayList<>(configManager.getNodeManager().getRegisteredConfigNodes());
- configNodeLocations.add(tConfigNodeLocation);
- TSStatus status =
- (TSStatus)
- SyncConfigNodeClientPool.getInstance()
- .sendSyncRequestToConfigNodeWithRetry(
- tConfigNodeLocation.getInternalEndPoint(),
- new TAddConsensusGroupReq(configNodeLocations),
- CnToCnNodeRequestType.ADD_CONSENSUS_GROUP);
- if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new AddConsensusGroupException(tConfigNodeLocation);
- }
- }
-
- /**
- * Leader will add the new ConfigNode Peer into ConfigRegion.
- *
- * @param configNodeLocation The new ConfigNode
- * @throws AddPeerException When addPeer doesn't success
- */
- public void addConfigNodePeer(TConfigNodeLocation configNodeLocation) throws AddPeerException {
- configManager.getConsensusManager().addConfigNodePeer(configNodeLocation);
- }
-
- /**
- * Remove peer in Leader node.
- *
- * @param tConfigNodeLocation node is removed
- * @throws ProcedureException if failed status
- */
- public void removeConfigNodePeer(TConfigNodeLocation tConfigNodeLocation)
- throws ProcedureException {
- removeConfigNodeLock.lock();
- TSStatus tsStatus;
- try {
- // Execute removePeer
- if (getConsensusManager().removeConfigNodePeer(tConfigNodeLocation)) {
- tsStatus = getConsensusManager().write(new RemoveConfigNodePlan(tConfigNodeLocation));
- } else {
- tsStatus =
- new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode())
- .setMessage(
- "Remove ConfigNode failed because update ConsensusGroup peer information failed.");
- }
- } catch (ConsensusException e) {
- LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
- tsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
- tsStatus.setMessage(e.getMessage());
- }
- try {
- if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new ProcedureException(tsStatus.getMessage());
- }
- } finally {
- removeConfigNodeLock.unlock();
- }
- }
-
- /**
- * Remove Consensus Group in removed node.
- *
- * @param removedConfigNode config node location
- * @throws ProcedureException if failed status
- */
- public void deleteConfigNodePeer(TConfigNodeLocation removedConfigNode)
- throws ProcedureException {
- TSStatus tsStatus =
- (TSStatus)
- SyncConfigNodeClientPool.getInstance()
- .sendSyncRequestToConfigNodeWithRetry(
- removedConfigNode.getInternalEndPoint(),
- removedConfigNode,
- CnToCnNodeRequestType.DELETE_CONFIG_NODE_PEER);
- if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new ProcedureException(tsStatus.getMessage());
- }
- }
-
- /**
- * Stop ConfigNode and remove heartbeatCache.
- *
- * @param tConfigNodeLocation config node location
- * @throws ProcedureException if failed status
- */
- public void stopAndClearConfigNode(TConfigNodeLocation tConfigNodeLocation)
- throws ProcedureException {
- TSStatus tsStatus =
- (TSStatus)
- SyncConfigNodeClientPool.getInstance()
- .sendSyncRequestToConfigNodeWithRetry(
- tConfigNodeLocation.getInternalEndPoint(),
- tConfigNodeLocation,
- CnToCnNodeRequestType.STOP_AND_CLEAR_CONFIG_NODE);
-
- if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- throw new ProcedureException(tsStatus.getMessage());
- }
-
- getLoadManager().removeNodeCache(tConfigNodeLocation.getConfigNodeId());
- }
-
- /**
- * Leader will record the new ConfigNode's information.
- *
- * @param configNodeLocation The new ConfigNode
- * @param versionInfo The new ConfigNode's versionInfo
- */
- public void applyConfigNode(
- TConfigNodeLocation configNodeLocation, TNodeVersionInfo versionInfo) {
- configManager.getNodeManager().applyConfigNode(configNodeLocation, versionInfo);
- }
-
- /**
- * Leader will notify the new ConfigNode that registration success.
- *
- * @param configNodeLocation The new ConfigNode
- */
- public void notifyRegisterSuccess(TConfigNodeLocation configNodeLocation) {
- SyncConfigNodeClientPool.getInstance()
- .sendSyncRequestToConfigNodeWithRetry(
- configNodeLocation.getInternalEndPoint(),
- null,
- CnToCnNodeRequestType.NOTIFY_REGISTER_SUCCESS);
- }
-
- /**
- * Create a new ConfigNodeHeartbeatCache
- *
- * @param nodeId the index of the new ConfigNode
- */
- public void createConfigNodeHeartbeatCache(int nodeId) {
- getLoadManager().getLoadCache().createNodeHeartbeatCache(NodeType.ConfigNode, nodeId);
- // TODO: invoke a force heartbeat to update new ConfigNode's status immediately
- }
-
- /**
- * Do region creations and broadcast the {@link CreateRegionGroupsPlan}.
- *
- * @return Those RegionReplicas that failed to create
- */
- public Map<TConsensusGroupId, TRegionReplicaSet> doRegionCreation(
- final TConsensusGroupType consensusGroupType,
- final CreateRegionGroupsPlan createRegionGroupsPlan) {
-
- // Prepare clientHandler
- DataNodeAsyncRequestContext<?, TSStatus> clientHandler;
- switch (consensusGroupType) {
- case SchemaRegion:
- clientHandler = getCreateSchemaRegionClientHandler(createRegionGroupsPlan);
- break;
- case DataRegion:
- default:
- clientHandler = getCreateDataRegionClientHandler(createRegionGroupsPlan);
- break;
- }
- if (clientHandler.getRequestIndices().isEmpty()) {
- return new HashMap<>();
- }
-
- // Send CreateRegion requests to DataNodes
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
-
- // Filter RegionGroups that weren't created successfully
- int requestId = 0;
- Map<Integer, TSStatus> responseMap = clientHandler.getResponseMap();
- Map<TConsensusGroupId, TRegionReplicaSet> failedRegions = new HashMap<>();
- for (List<TRegionReplicaSet> regionReplicaSets :
- createRegionGroupsPlan.getRegionGroupMap().values()) {
- for (final TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
- for (final TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
- if (responseMap.get(requestId).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- failedRegions
- .computeIfAbsent(
- regionReplicaSet.getRegionId(),
- empty -> new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()))
- .addToDataNodeLocations(dataNodeLocation);
- }
- requestId += 1;
- }
- }
- }
- return failedRegions;
- }
-
- private DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus>
- getCreateSchemaRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
- DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_SCHEMA_REGION);
-
- int requestId = 0;
- for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
- createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
- String storageGroup = sgRegionsEntry.getKey();
- List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
- for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
- for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
- clientHandler.putRequest(
- requestId, genCreateSchemaRegionReq(storageGroup, regionReplicaSet));
- clientHandler.putNodeLocation(requestId, dataNodeLocation);
- requestId += 1;
- }
- }
- }
-
- return clientHandler;
- }
-
- private DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus>
- getCreateDataRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
- DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_DATA_REGION);
-
- int requestId = 0;
- for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
- createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
- String storageGroup = sgRegionsEntry.getKey();
- List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
- for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
- for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
- clientHandler.putRequest(
- requestId, genCreateDataRegionReq(storageGroup, regionReplicaSet));
- clientHandler.putNodeLocation(requestId, dataNodeLocation);
- requestId += 1;
- }
- }
- }
-
- return clientHandler;
- }
-
- private TCreateSchemaRegionReq genCreateSchemaRegionReq(
- String storageGroup, TRegionReplicaSet regionReplicaSet) {
- TCreateSchemaRegionReq req = new TCreateSchemaRegionReq();
- req.setStorageGroup(storageGroup);
- req.setRegionReplicaSet(regionReplicaSet);
- return req;
- }
-
- private TCreateDataRegionReq genCreateDataRegionReq(
- String storageGroup, TRegionReplicaSet regionReplicaSet) {
- TCreateDataRegionReq req = new TCreateDataRegionReq();
- req.setStorageGroup(storageGroup);
- req.setRegionReplicaSet(regionReplicaSet);
- return req;
- }
-
- public List<TSStatus> notifyRegionMigrationToAllDataNodes(
- TConsensusGroupId consensusGroupId, boolean isStart) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TNotifyRegionMigrationReq request =
- new TNotifyRegionMigrationReq(
- configManager
- .getConsensusManager()
- .getConsensusImpl()
- .getLogicalClock(ConfigNodeInfo.CONFIG_REGION_ID),
- System.nanoTime(),
- configManager.getProcedureManager().getRegionOperationConsensusIds());
- request.setRegionId(consensusGroupId);
- request.setIsStart(isStart);
-
- final DataNodeAsyncRequestContext<TNotifyRegionMigrationReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.NOTIFY_REGION_MIGRATION, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public void persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
- // Persist the allocation result
- try {
- getConsensusManager().write(createRegionGroupsPlan);
- } catch (ConsensusException e) {
- LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
- }
- }
-
- /**
- * Force activating RegionGroup by setting status to Running, therefore the ConfigNode-leader can
- * select leader for it and use it to allocate new Partitions
- *
- * @param activateRegionGroupMap Map<Database, Map<RegionGroupId, Map<DataNodeId, activate
- * heartbeat sample>>>
- */
- public void activateRegionGroup(
- Map<String, Map<TConsensusGroupId, Map<Integer, RegionHeartbeatSample>>>
- activateRegionGroupMap) {
- // Create RegionGroup heartbeat Caches
- activateRegionGroupMap.forEach(
- (database, regionGroupSampleMap) ->
- regionGroupSampleMap.forEach(
- (regionGroupId, regionSampleMap) ->
- getLoadManager()
- .getLoadCache()
- .createRegionGroupHeartbeatCache(
- database, regionGroupId, regionSampleMap.keySet())));
- // Force update first heartbeat samples
- getLoadManager()
- .forceUpdateRegionGroupCache(
- activateRegionGroupMap.values().stream()
- .flatMap(innerMap -> innerMap.entrySet().stream())
- .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b)));
- // Wait for leader and priority redistribution
- getLoadManager()
- .waitForRegionGroupReady(
- activateRegionGroupMap.values().stream()
- .flatMap(innterMap -> innterMap.keySet().stream())
- .collect(Collectors.toList()));
- }
-
- public List<TRegionReplicaSet> getAllReplicaSets(String storageGroup) {
- return getPartitionManager().getAllReplicaSets(storageGroup);
- }
-
- public List<TSStatus> createTriggerOnDataNodes(
- TriggerInformation triggerInformation, Binary jarFile) throws IOException {
- NodeManager nodeManager = configManager.getNodeManager();
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- nodeManager.getRegisteredDataNodeLocations();
- final TCreateTriggerInstanceReq request =
- new TCreateTriggerInstanceReq(triggerInformation.serialize());
- if (jarFile != null) {
- request.setJarFile(ByteBuffer.wrap(jarFile.getValues()));
- }
-
- DataNodeAsyncRequestContext<TCreateTriggerInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.CREATE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public List<TSStatus> dropTriggerOnDataNodes(String triggerName, boolean needToDeleteJarFile) {
- NodeManager nodeManager = configManager.getNodeManager();
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- nodeManager.getRegisteredDataNodeLocations();
- final TDropTriggerInstanceReq request =
- new TDropTriggerInstanceReq(triggerName, needToDeleteJarFile);
-
- DataNodeAsyncRequestContext<TDropTriggerInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.DROP_TRIGGER_INSTANCE, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public List<TSStatus> activeTriggerOnDataNodes(String triggerName) {
- NodeManager nodeManager = configManager.getNodeManager();
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- nodeManager.getRegisteredDataNodeLocations();
- final TActiveTriggerInstanceReq request = new TActiveTriggerInstanceReq(triggerName);
-
- DataNodeAsyncRequestContext<TActiveTriggerInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.ACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public List<TSStatus> inactiveTriggerOnDataNodes(String triggerName) {
- NodeManager nodeManager = configManager.getNodeManager();
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- nodeManager.getRegisteredDataNodeLocations();
- final TInactiveTriggerInstanceReq request = new TInactiveTriggerInstanceReq(triggerName);
-
- DataNodeAsyncRequestContext<TInactiveTriggerInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.INACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public List<TSStatus> createPipePluginOnDataNodes(PipePluginMeta pipePluginMeta, byte[] jarFile)
- throws IOException {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TCreatePipePluginInstanceReq request =
- new TCreatePipePluginInstanceReq(pipePluginMeta.serialize(), ByteBuffer.wrap(jarFile));
-
- final DataNodeAsyncRequestContext<TCreatePipePluginInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.CREATE_PIPE_PLUGIN, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public List<TSStatus> dropPipePluginOnDataNodes(
- String pipePluginName, boolean needToDeleteJarFile) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TDropPipePluginInstanceReq request =
- new TDropPipePluginInstanceReq(pipePluginName, needToDeleteJarFile);
-
- DataNodeAsyncRequestContext<TDropPipePluginInstanceReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.DROP_PIPE_PLUGIN, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList();
- }
-
- public Map<Integer, TPushPipeMetaResp> pushAllPipeMetaToDataNodes(
- List<ByteBuffer> pipeMetaBinaryList) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushPipeMetaReq request = new TPushPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
-
- final DataNodeAsyncRequestContext<TPushPipeMetaReq, TPushPipeMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.PIPE_PUSH_ALL_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushPipeMetaResp> pushSinglePipeMetaToDataNodes(ByteBuffer pipeMetaBinary) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSinglePipeMetaReq request = new TPushSinglePipeMetaReq().setPipeMeta(pipeMetaBinary);
-
- final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushPipeMetaResp> dropSinglePipeOnDataNodes(String pipeNameToDrop) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSinglePipeMetaReq request =
- new TPushSinglePipeMetaReq().setPipeNameToDrop(pipeNameToDrop);
-
- final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushPipeMetaResp> pushMultiPipeMetaToDataNodes(
- List<ByteBuffer> pipeMetaBinaryList) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushMultiPipeMetaReq request =
- new TPushMultiPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
-
- final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushPipeMetaResp> dropMultiPipeOnDataNodes(List<String> pipeNamesToDrop) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushMultiPipeMetaReq request =
- new TPushMultiPipeMetaReq().setPipeNamesToDrop(pipeNamesToDrop);
-
- final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushTopicMetaResp> pushAllTopicMetaToDataNodes(
- List<ByteBuffer> topicMetaBinaryList) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushTopicMetaReq request = new TPushTopicMetaReq().setTopicMetas(topicMetaBinaryList);
-
- final DataNodeAsyncRequestContext<TPushTopicMetaReq, TPushTopicMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.TOPIC_PUSH_ALL_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public List<TSStatus> pushSingleTopicOnDataNode(ByteBuffer topicMeta) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSingleTopicMetaReq request = new TPushSingleTopicMetaReq().setTopicMeta(topicMeta);
-
- final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList().stream()
- .map(TPushTopicMetaResp::getStatus)
- .collect(Collectors.toList());
- }
-
- public List<TSStatus> dropSingleTopicOnDataNode(String topicNameToDrop) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSingleTopicMetaReq request =
- new TPushSingleTopicMetaReq().setTopicNameToDrop(topicNameToDrop);
-
- final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList().stream()
- .map(TPushTopicMetaResp::getStatus)
- .collect(Collectors.toList());
- }
-
- public Map<Integer, TPushTopicMetaResp> pushMultiTopicMetaToDataNodes(
- List<ByteBuffer> topicMetaBinaryList) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushMultiTopicMetaReq request =
- new TPushMultiTopicMetaReq().setTopicMetas(topicMetaBinaryList);
-
- final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushTopicMetaResp> dropMultiTopicOnDataNodes(List<String> topicNamesToDrop) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushMultiTopicMetaReq request =
- new TPushMultiTopicMetaReq().setTopicNamesToDrop(topicNamesToDrop);
-
- final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public Map<Integer, TPushConsumerGroupMetaResp> pushAllConsumerGroupMetaToDataNodes(
- List<ByteBuffer> consumerGroupMetaBinaryList) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushConsumerGroupMetaReq request =
- new TPushConsumerGroupMetaReq().setConsumerGroupMetas(consumerGroupMetaBinaryList);
-
- final DataNodeAsyncRequestContext<TPushConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
- clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_ALL_META, request, dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance()
- .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
- clientHandler,
- PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
- return clientHandler.getResponseMap();
- }
-
- public List<TSStatus> pushSingleConsumerGroupOnDataNode(ByteBuffer consumerGroupMeta) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSingleConsumerGroupMetaReq request =
- new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(consumerGroupMeta);
-
- final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
- clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
- request,
- dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList().stream()
- .map(TPushConsumerGroupMetaResp::getStatus)
- .collect(Collectors.toList());
- }
-
- public List<TSStatus> dropSingleConsumerGroupOnDataNode(String consumerGroupNameToDrop) {
- final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
- configManager.getNodeManager().getRegisteredDataNodeLocations();
- final TPushSingleConsumerGroupMetaReq request =
- new TPushSingleConsumerGroupMetaReq().setConsumerGroupNameToDrop(consumerGroupNameToDrop);
-
- final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
- clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
- request,
- dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseList().stream()
- .map(TPushConsumerGroupMetaResp::getStatus)
- .collect(Collectors.toList());
- }
-
- public LockQueue getNodeLock() {
- return nodeLock;
- }
-
- public ProcedureScheduler getScheduler() {
- return scheduler;
- }
-
- public ReentrantLock getSchedulerLock() {
- return schedulerLock;
- }
-
- public ReentrantLock getSubmitRegionMigrateLock() {
- return submitRegionMigrateLock;
- }
-
- public RegionMaintainHandler getRegionMaintainHandler() {
- return regionMaintainHandler;
- }
-
- public RemoveDataNodeHandler getRemoveDataNodeHandler() {
- return removeDataNodeHandler;
- }
-
- private ConsensusManager getConsensusManager() {
- return configManager.getConsensusManager();
- }
-
- private NodeManager getNodeManager() {
- return configManager.getNodeManager();
- }
-
- private ClusterSchemaManager getClusterSchemaManager() {
- return configManager.getClusterSchemaManager();
- }
-
- private PartitionManager getPartitionManager() {
- return configManager.getPartitionManager();
- }
-
- private LoadManager getLoadManager() {
- return configManager.getLoadManager();
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1
+1package org.apache.iotdb.confignode.procedure.env;
+1
+1import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+1import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+1import org.apache.iotdb.common.rpc.thrift.TSStatus;
+1import org.apache.iotdb.commons.cluster.NodeStatus;
+1import org.apache.iotdb.commons.cluster.NodeType;
+1import org.apache.iotdb.commons.pipe.agent.plugin.meta.PipePluginMeta;
+1import org.apache.iotdb.commons.pipe.config.PipeConfig;
+1import org.apache.iotdb.commons.trigger.TriggerInformation;
+1import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
+1import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+1import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+1import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
+1import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
+1import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
+1import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
+1import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.database.DeleteDatabasePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.database.PreDeleteDatabasePlan;
+1import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
+1import org.apache.iotdb.confignode.exception.AddConsensusGroupException;
+1import org.apache.iotdb.confignode.exception.AddPeerException;
+1import org.apache.iotdb.confignode.manager.ConfigManager;
+1import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+1import org.apache.iotdb.confignode.manager.load.LoadManager;
+1import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
+1import org.apache.iotdb.confignode.manager.node.NodeManager;
+1import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+1import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
+1import org.apache.iotdb.confignode.persistence.partition.PartitionInfo;
+1import org.apache.iotdb.confignode.persistence.schema.ClusterSchemaInfo;
+1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+1import org.apache.iotdb.confignode.procedure.scheduler.LockQueue;
+1import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+1import org.apache.iotdb.confignode.rpc.thrift.TAddConsensusGroupReq;
+1import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
+1import org.apache.iotdb.consensus.exception.ConsensusException;
+1import org.apache.iotdb.db.protocol.client.ConfigNodeInfo;
+1import org.apache.iotdb.mpp.rpc.thrift.TActiveTriggerInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TCreateDataRegionReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TCreatePipePluginInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TCreateSchemaRegionReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TCreateTriggerInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TDropPipePluginInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TDropTriggerInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TInactiveTriggerInstanceReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TInvalidateCacheReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TNotifyRegionMigrationReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushConsumerGroupMetaResp;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushMultiPipeMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushMultiTopicMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushPipeMetaResp;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushSingleConsumerGroupMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushSinglePipeMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushSingleTopicMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TPushTopicMetaResp;
+1import org.apache.iotdb.rpc.TSStatusCode;
+1
+1import org.apache.thrift.TException;
+1import org.apache.tsfile.utils.Binary;
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.io.IOException;
+1import java.nio.ByteBuffer;
+1import java.util.ArrayList;
+1import java.util.Arrays;
+1import java.util.HashMap;
+1import java.util.List;
+1import java.util.Map;
+1import java.util.concurrent.TimeUnit;
+1import java.util.concurrent.locks.ReentrantLock;
+1import java.util.stream.Collectors;
+1
+1public class ConfigNodeProcedureEnv {
+1
+1 private static final Logger LOG = LoggerFactory.getLogger(ConfigNodeProcedureEnv.class);
+1
+1 /** Add or remove node lock. */
+1 private final LockQueue nodeLock = new LockQueue();
+1
+1 private final ReentrantLock schedulerLock = new ReentrantLock(true);
+1
+1 private final ReentrantLock submitRegionMigrateLock = new ReentrantLock(true);
+1
+1 private final ConfigManager configManager;
+1
+1 private final ProcedureScheduler scheduler;
+1
+1 private final RegionMaintainHandler regionMaintainHandler;
+1
+1 private final RemoveDataNodeHandler removeDataNodeHandler;
+1
+1 private final ReentrantLock removeConfigNodeLock;
+1
+1 public ConfigNodeProcedureEnv(ConfigManager configManager, ProcedureScheduler scheduler) {
+1 this.configManager = configManager;
+1 this.scheduler = scheduler;
+1 this.regionMaintainHandler = new RegionMaintainHandler(configManager);
+1 this.removeDataNodeHandler = new RemoveDataNodeHandler(configManager);
+1 this.removeConfigNodeLock = new ReentrantLock();
+1 }
+1
+1 public ConfigManager getConfigManager() {
+1 return configManager;
+1 }
+1
+1 /**
+1 * Delete ConfigNode cache, includes {@link ClusterSchemaInfo} and {@link PartitionInfo}.
+1 *
+1 * @param name database name
+1 * @param isGeneratedByPipe whether the deletion is triggered by pipe request
+1 * @return tsStatus
+1 */
+1 public TSStatus deleteDatabaseConfig(final String name, final boolean isGeneratedByPipe) {
+1 return getClusterSchemaManager()
+1 .deleteDatabase(new DeleteDatabasePlan(name), isGeneratedByPipe);
+1 }
+1
+1 /**
+1 * Pre delete a database.
+1 *
+1 * @param preDeleteType execute/rollback
+1 * @param deleteSgName database name
+1 */
+1 public void preDeleteDatabase(
+1 final PreDeleteDatabasePlan.PreDeleteType preDeleteType, final String deleteSgName) {
+1 getPartitionManager().preDeleteDatabase(deleteSgName, preDeleteType);
+1 }
+1
+1 /**
+1 * @param databaseName database name
+1 * @return ALL SUCCESS OR NOT
+1 * @throws IOException IOE
+1 * @throws TException Thrift IOE
+1 */
+1 public boolean invalidateCache(final String databaseName) throws IOException, TException {
+1 final List<TDataNodeConfiguration> allDataNodes = getNodeManager().getRegisteredDataNodes();
+1 final TInvalidateCacheReq invalidateCacheReq = new TInvalidateCacheReq();
+1 invalidateCacheReq.setStorageGroup(true);
+1 invalidateCacheReq.setFullPath(databaseName);
+1 for (final TDataNodeConfiguration dataNodeConfiguration : allDataNodes) {
+1 final int dataNodeId = dataNodeConfiguration.getLocation().getDataNodeId();
+1
+1 // If the node is not alive, retry for up to 10 times
+1 NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
+1 int retryNum = 10;
+1 if (nodeStatus == NodeStatus.Unknown) {
+1 for (int i = 0; i < retryNum && nodeStatus == NodeStatus.Unknown; i++) {
+1 try {
+1 TimeUnit.MILLISECONDS.sleep(500);
+1 } catch (final InterruptedException e) {
+1 LOG.error("Sleep failed in ConfigNodeProcedureEnv: ", e);
+1 Thread.currentThread().interrupt();
+1 break;
+1 }
+1 nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
+1 }
+1 }
+1
+1 if (nodeStatus == NodeStatus.Unknown) {
+1 LOG.warn(
+1 "Invalidate cache failed, because DataNode {} is Unknown",
+1 dataNodeConfiguration.getLocation().getInternalEndPoint());
+1 return false;
+1 }
+1
+1 // Always invalidate PartitionCache first
+1 final TSStatus invalidatePartitionStatus =
+1 (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 dataNodeConfiguration.getLocation().getInternalEndPoint(),
+1 invalidateCacheReq,
+1 CnToDnSyncRequestType.INVALIDATE_PARTITION_CACHE);
+1
+1 final TSStatus invalidateSchemaStatus =
+1 (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 dataNodeConfiguration.getLocation().getInternalEndPoint(),
+1 invalidateCacheReq,
+1 CnToDnSyncRequestType.INVALIDATE_SCHEMA_CACHE);
+1
+1 if (!verifySucceed(invalidatePartitionStatus, invalidateSchemaStatus)) {
+1 LOG.error(
+1 "Invalidate cache failed, invalidate partition cache status is {}, invalidate schemaengine cache status is {}",
+1 invalidatePartitionStatus,
+1 invalidateSchemaStatus);
+1 return false;
+1 }
+1 }
+1 return true;
+1 }
+1
+1 public boolean verifySucceed(TSStatus... status) {
+1 return Arrays.stream(status)
+1 .allMatch(tsStatus -> tsStatus.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode());
+1 }
+1
+1 /**
+1 * Let the remotely new ConfigNode build the ConsensusGroup.
+1 *
+1 * <p>Actually, the parameter of this method can be empty, adding new raft peer to exist group
+1 * should invoke createPeer(groupId, emptyList).
+1 *
+1 * @param tConfigNodeLocation New ConfigNode's location
+1 */
+1 public void addConsensusGroup(TConfigNodeLocation tConfigNodeLocation)
+1 throws AddConsensusGroupException {
+1 List<TConfigNodeLocation> configNodeLocations =
+1 new ArrayList<>(configManager.getNodeManager().getRegisteredConfigNodes());
+1 configNodeLocations.add(tConfigNodeLocation);
+1 TSStatus status =
+1 (TSStatus)
+1 SyncConfigNodeClientPool.getInstance()
+1 .sendSyncRequestToConfigNodeWithRetry(
+1 tConfigNodeLocation.getInternalEndPoint(),
+1 new TAddConsensusGroupReq(configNodeLocations),
+1 CnToCnNodeRequestType.ADD_CONSENSUS_GROUP);
+1 if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+1 throw new AddConsensusGroupException(tConfigNodeLocation);
+1 }
+1 }
+1
+1 /**
+1 * Leader will add the new ConfigNode Peer into ConfigRegion.
+1 *
+1 * @param configNodeLocation The new ConfigNode
+1 * @throws AddPeerException When addPeer doesn't success
+1 */
+1 public void addConfigNodePeer(TConfigNodeLocation configNodeLocation) throws AddPeerException {
+1 configManager.getConsensusManager().addConfigNodePeer(configNodeLocation);
+1 }
+1
+1 /**
+1 * Remove peer in Leader node.
+1 *
+1 * @param tConfigNodeLocation node is removed
+1 * @throws ProcedureException if failed status
+1 */
+1 public void removeConfigNodePeer(TConfigNodeLocation tConfigNodeLocation)
+1 throws ProcedureException {
+1 removeConfigNodeLock.lock();
+1 TSStatus tsStatus;
+1 try {
+1 // Execute removePeer
+1 if (getConsensusManager().removeConfigNodePeer(tConfigNodeLocation)) {
+1 tsStatus = getConsensusManager().write(new RemoveConfigNodePlan(tConfigNodeLocation));
+1 } else {
+1 tsStatus =
+1 new TSStatus(TSStatusCode.REMOVE_CONFIGNODE_ERROR.getStatusCode())
+1 .setMessage(
+1 "Remove ConfigNode failed because update ConsensusGroup peer information failed.");
+1 }
+1 } catch (ConsensusException e) {
+1 LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
+1 tsStatus = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+1 tsStatus.setMessage(e.getMessage());
+1 }
+1 try {
+1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+1 throw new ProcedureException(tsStatus.getMessage());
+1 }
+1 } finally {
+1 removeConfigNodeLock.unlock();
+1 }
+1 }
+1
+1 /**
+1 * Remove Consensus Group in removed node.
+1 *
+1 * @param removedConfigNode config node location
+1 * @throws ProcedureException if failed status
+1 */
+1 public void deleteConfigNodePeer(TConfigNodeLocation removedConfigNode)
+1 throws ProcedureException {
+1 TSStatus tsStatus =
+1 (TSStatus)
+1 SyncConfigNodeClientPool.getInstance()
+1 .sendSyncRequestToConfigNodeWithRetry(
+1 removedConfigNode.getInternalEndPoint(),
+1 removedConfigNode,
+1 CnToCnNodeRequestType.DELETE_CONFIG_NODE_PEER);
+1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+1 throw new ProcedureException(tsStatus.getMessage());
+1 }
+1 }
+1
+1 /**
+1 * Stop ConfigNode and remove heartbeatCache.
+1 *
+1 * @param tConfigNodeLocation config node location
+1 * @throws ProcedureException if failed status
+1 */
+1 public void stopAndClearConfigNode(TConfigNodeLocation tConfigNodeLocation)
+1 throws ProcedureException {
+1 TSStatus tsStatus =
+1 (TSStatus)
+1 SyncConfigNodeClientPool.getInstance()
+1 .sendSyncRequestToConfigNodeWithRetry(
+1 tConfigNodeLocation.getInternalEndPoint(),
+1 tConfigNodeLocation,
+1 CnToCnNodeRequestType.STOP_AND_CLEAR_CONFIG_NODE);
+1
+1 if (tsStatus.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+1 throw new ProcedureException(tsStatus.getMessage());
+1 }
+1
+1 getLoadManager().removeNodeCache(tConfigNodeLocation.getConfigNodeId());
+1 }
+1
+1 /**
+1 * Leader will record the new ConfigNode's information.
+1 *
+1 * @param configNodeLocation The new ConfigNode
+1 * @param versionInfo The new ConfigNode's versionInfo
+1 */
+1 public void applyConfigNode(
+1 TConfigNodeLocation configNodeLocation, TNodeVersionInfo versionInfo) {
+1 configManager.getNodeManager().applyConfigNode(configNodeLocation, versionInfo);
+1 }
+1
+1 /**
+1 * Leader will notify the new ConfigNode that registration success.
+1 *
+1 * @param configNodeLocation The new ConfigNode
+1 */
+1 public void notifyRegisterSuccess(TConfigNodeLocation configNodeLocation) {
+1 SyncConfigNodeClientPool.getInstance()
+1 .sendSyncRequestToConfigNodeWithRetry(
+1 configNodeLocation.getInternalEndPoint(),
+1 null,
+1 CnToCnNodeRequestType.NOTIFY_REGISTER_SUCCESS);
+1 }
+1
+1 /**
+1 * Create a new ConfigNodeHeartbeatCache
+1 *
+1 * @param nodeId the index of the new ConfigNode
+1 */
+1 public void createConfigNodeHeartbeatCache(int nodeId) {
+1 getLoadManager().getLoadCache().createNodeHeartbeatCache(NodeType.ConfigNode, nodeId);
+1 // TODO: invoke a force heartbeat to update new ConfigNode's status immediately
+1 }
+1
+1 /**
+1 * Do region creations and broadcast the {@link CreateRegionGroupsPlan}.
+1 *
+1 * @return Those RegionReplicas that failed to create
+1 */
+1 public Map<TConsensusGroupId, TRegionReplicaSet> doRegionCreation(
+1 final TConsensusGroupType consensusGroupType,
+1 final CreateRegionGroupsPlan createRegionGroupsPlan) {
+1
+1 // Prepare clientHandler
+1 DataNodeAsyncRequestContext<?, TSStatus> clientHandler;
+1 switch (consensusGroupType) {
+1 case SchemaRegion:
+1 clientHandler = getCreateSchemaRegionClientHandler(createRegionGroupsPlan);
+1 break;
+1 case DataRegion:
+1 default:
+1 clientHandler = getCreateDataRegionClientHandler(createRegionGroupsPlan);
+1 break;
+1 }
+1 if (clientHandler.getRequestIndices().isEmpty()) {
+1 return new HashMap<>();
+1 }
+1
+1 // Send CreateRegion requests to DataNodes
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1
+1 // Filter RegionGroups that weren't created successfully
+1 int requestId = 0;
+1 Map<Integer, TSStatus> responseMap = clientHandler.getResponseMap();
+1 Map<TConsensusGroupId, TRegionReplicaSet> failedRegions = new HashMap<>();
+1 for (List<TRegionReplicaSet> regionReplicaSets :
+1 createRegionGroupsPlan.getRegionGroupMap().values()) {
+1 for (final TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+1 for (final TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+1 if (responseMap.get(requestId).getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+1 failedRegions
+1 .computeIfAbsent(
+1 regionReplicaSet.getRegionId(),
+1 empty -> new TRegionReplicaSet().setRegionId(regionReplicaSet.getRegionId()))
+1 .addToDataNodeLocations(dataNodeLocation);
+1 }
+1 requestId += 1;
+1 }
+1 }
+1 }
+1 return failedRegions;
+1 }
+1
+1 private DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus>
+1 getCreateSchemaRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
+1 DataNodeAsyncRequestContext<TCreateSchemaRegionReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_SCHEMA_REGION);
+1
+1 int requestId = 0;
+1 for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
+1 createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
+1 String storageGroup = sgRegionsEntry.getKey();
+1 List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
+1 for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+1 for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+1 clientHandler.putRequest(
+1 requestId, genCreateSchemaRegionReq(storageGroup, regionReplicaSet));
+1 clientHandler.putNodeLocation(requestId, dataNodeLocation);
+1 requestId += 1;
+1 }
+1 }
+1 }
+1
+1 return clientHandler;
+1 }
+1
+1 private DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus>
+1 getCreateDataRegionClientHandler(CreateRegionGroupsPlan createRegionGroupsPlan) {
+1 DataNodeAsyncRequestContext<TCreateDataRegionReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(CnToDnAsyncRequestType.CREATE_DATA_REGION);
+1
+1 int requestId = 0;
+1 for (Map.Entry<String, List<TRegionReplicaSet>> sgRegionsEntry :
+1 createRegionGroupsPlan.getRegionGroupMap().entrySet()) {
+1 String storageGroup = sgRegionsEntry.getKey();
+1 List<TRegionReplicaSet> regionReplicaSets = sgRegionsEntry.getValue();
+1 for (TRegionReplicaSet regionReplicaSet : regionReplicaSets) {
+1 for (TDataNodeLocation dataNodeLocation : regionReplicaSet.getDataNodeLocations()) {
+1 clientHandler.putRequest(
+1 requestId, genCreateDataRegionReq(storageGroup, regionReplicaSet));
+1 clientHandler.putNodeLocation(requestId, dataNodeLocation);
+1 requestId += 1;
+1 }
+1 }
+1 }
+1
+1 return clientHandler;
+1 }
+1
+1 private TCreateSchemaRegionReq genCreateSchemaRegionReq(
+1 String storageGroup, TRegionReplicaSet regionReplicaSet) {
+1 TCreateSchemaRegionReq req = new TCreateSchemaRegionReq();
+1 req.setStorageGroup(storageGroup);
+1 req.setRegionReplicaSet(regionReplicaSet);
+1 return req;
+1 }
+1
+1 private TCreateDataRegionReq genCreateDataRegionReq(
+1 String storageGroup, TRegionReplicaSet regionReplicaSet) {
+1 TCreateDataRegionReq req = new TCreateDataRegionReq();
+1 req.setStorageGroup(storageGroup);
+1 req.setRegionReplicaSet(regionReplicaSet);
+1 return req;
+1 }
+1
+1 public List<TSStatus> notifyRegionMigrationToAllDataNodes(
+1 TConsensusGroupId consensusGroupId, boolean isStart) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TNotifyRegionMigrationReq request =
+1 new TNotifyRegionMigrationReq(
+1 configManager
+1 .getConsensusManager()
+1 .getConsensusImpl()
+1 .getLogicalClock(ConfigNodeInfo.CONFIG_REGION_ID),
+1 System.nanoTime(),
+1 configManager.getProcedureManager().getRegionOperationConsensusIds());
+1 request.setRegionId(consensusGroupId);
+1 request.setIsStart(isStart);
+1
+1 final DataNodeAsyncRequestContext<TNotifyRegionMigrationReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.NOTIFY_REGION_MIGRATION, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public void persistRegionGroup(CreateRegionGroupsPlan createRegionGroupsPlan) {
+1 // Persist the allocation result
+1 try {
+1 getConsensusManager().write(createRegionGroupsPlan);
+1 } catch (ConsensusException e) {
+1 LOG.warn("Failed in the write API executing the consensus layer due to: ", e);
+1 }
+1 }
+1
+1 /**
+1 * Force activating RegionGroup by setting status to Running, therefore the ConfigNode-leader can
+1 * select leader for it and use it to allocate new Partitions
+1 *
+1 * @param activateRegionGroupMap Map<Database, Map<RegionGroupId, Map<DataNodeId, activate
+1 * heartbeat sample>>>
+1 */
+1 public void activateRegionGroup(
+1 Map<String, Map<TConsensusGroupId, Map<Integer, RegionHeartbeatSample>>>
+1 activateRegionGroupMap) {
+1 // Create RegionGroup heartbeat Caches
+1 activateRegionGroupMap.forEach(
+1 (database, regionGroupSampleMap) ->
+1 regionGroupSampleMap.forEach(
+1 (regionGroupId, regionSampleMap) ->
+1 getLoadManager()
+1 .getLoadCache()
+1 .createRegionGroupHeartbeatCache(
+1 database, regionGroupId, regionSampleMap.keySet())));
+1 // Force update first heartbeat samples
+1 getLoadManager()
+1 .forceUpdateRegionGroupCache(
+1 activateRegionGroupMap.values().stream()
+1 .flatMap(innerMap -> innerMap.entrySet().stream())
+1 .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (a, b) -> b)));
+1 // Wait for leader and priority redistribution
+1 getLoadManager()
+1 .waitForRegionGroupReady(
+1 activateRegionGroupMap.values().stream()
+1 .flatMap(innterMap -> innterMap.keySet().stream())
+1 .collect(Collectors.toList()));
+1 }
+1
+1 public List<TRegionReplicaSet> getAllReplicaSets(String storageGroup) {
+1 return getPartitionManager().getAllReplicaSets(storageGroup);
+1 }
+1
+1 public List<TSStatus> createTriggerOnDataNodes(
+1 TriggerInformation triggerInformation, Binary jarFile) throws IOException {
+1 NodeManager nodeManager = configManager.getNodeManager();
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 nodeManager.getRegisteredDataNodeLocations();
+1 final TCreateTriggerInstanceReq request =
+1 new TCreateTriggerInstanceReq(triggerInformation.serialize());
+1 if (jarFile != null) {
+1 request.setJarFile(ByteBuffer.wrap(jarFile.getValues()));
+1 }
+1
+1 DataNodeAsyncRequestContext<TCreateTriggerInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.CREATE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public List<TSStatus> dropTriggerOnDataNodes(String triggerName, boolean needToDeleteJarFile) {
+1 NodeManager nodeManager = configManager.getNodeManager();
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 nodeManager.getRegisteredDataNodeLocations();
+1 final TDropTriggerInstanceReq request =
+1 new TDropTriggerInstanceReq(triggerName, needToDeleteJarFile);
+1
+1 DataNodeAsyncRequestContext<TDropTriggerInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.DROP_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public List<TSStatus> activeTriggerOnDataNodes(String triggerName) {
+1 NodeManager nodeManager = configManager.getNodeManager();
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 nodeManager.getRegisteredDataNodeLocations();
+1 final TActiveTriggerInstanceReq request = new TActiveTriggerInstanceReq(triggerName);
+1
+1 DataNodeAsyncRequestContext<TActiveTriggerInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.ACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public List<TSStatus> inactiveTriggerOnDataNodes(String triggerName) {
+1 NodeManager nodeManager = configManager.getNodeManager();
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 nodeManager.getRegisteredDataNodeLocations();
+1 final TInactiveTriggerInstanceReq request = new TInactiveTriggerInstanceReq(triggerName);
+1
+1 DataNodeAsyncRequestContext<TInactiveTriggerInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.INACTIVE_TRIGGER_INSTANCE, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public List<TSStatus> createPipePluginOnDataNodes(PipePluginMeta pipePluginMeta, byte[] jarFile)
+1 throws IOException {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TCreatePipePluginInstanceReq request =
+1 new TCreatePipePluginInstanceReq(pipePluginMeta.serialize(), ByteBuffer.wrap(jarFile));
+1
+1 final DataNodeAsyncRequestContext<TCreatePipePluginInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.CREATE_PIPE_PLUGIN, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public List<TSStatus> dropPipePluginOnDataNodes(
+1 String pipePluginName, boolean needToDeleteJarFile) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TDropPipePluginInstanceReq request =
+1 new TDropPipePluginInstanceReq(pipePluginName, needToDeleteJarFile);
+1
+1 DataNodeAsyncRequestContext<TDropPipePluginInstanceReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.DROP_PIPE_PLUGIN, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList();
+1 }
+1
+1 public Map<Integer, TPushPipeMetaResp> pushAllPipeMetaToDataNodes(
+1 List<ByteBuffer> pipeMetaBinaryList) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushPipeMetaReq request = new TPushPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
+1
+1 final DataNodeAsyncRequestContext<TPushPipeMetaReq, TPushPipeMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.PIPE_PUSH_ALL_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushPipeMetaResp> pushSinglePipeMetaToDataNodes(ByteBuffer pipeMetaBinary) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSinglePipeMetaReq request = new TPushSinglePipeMetaReq().setPipeMeta(pipeMetaBinary);
+1
+1 final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushPipeMetaResp> dropSinglePipeOnDataNodes(String pipeNameToDrop) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSinglePipeMetaReq request =
+1 new TPushSinglePipeMetaReq().setPipeNameToDrop(pipeNameToDrop);
+1
+1 final DataNodeAsyncRequestContext<TPushSinglePipeMetaReq, TPushPipeMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.PIPE_PUSH_SINGLE_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushPipeMetaResp> pushMultiPipeMetaToDataNodes(
+1 List<ByteBuffer> pipeMetaBinaryList) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushMultiPipeMetaReq request =
+1 new TPushMultiPipeMetaReq().setPipeMetas(pipeMetaBinaryList);
+1
+1 final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushPipeMetaResp> dropMultiPipeOnDataNodes(List<String> pipeNamesToDrop) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushMultiPipeMetaReq request =
+1 new TPushMultiPipeMetaReq().setPipeNamesToDrop(pipeNamesToDrop);
+1
+1 final DataNodeAsyncRequestContext<TPushMultiPipeMetaReq, TPushPipeMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.PIPE_PUSH_MULTI_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushTopicMetaResp> pushAllTopicMetaToDataNodes(
+1 List<ByteBuffer> topicMetaBinaryList) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushTopicMetaReq request = new TPushTopicMetaReq().setTopicMetas(topicMetaBinaryList);
+1
+1 final DataNodeAsyncRequestContext<TPushTopicMetaReq, TPushTopicMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.TOPIC_PUSH_ALL_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public List<TSStatus> pushSingleTopicOnDataNode(ByteBuffer topicMeta) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSingleTopicMetaReq request = new TPushSingleTopicMetaReq().setTopicMeta(topicMeta);
+1
+1 final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList().stream()
+1 .map(TPushTopicMetaResp::getStatus)
+1 .collect(Collectors.toList());
+1 }
+1
+1 public List<TSStatus> dropSingleTopicOnDataNode(String topicNameToDrop) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSingleTopicMetaReq request =
+1 new TPushSingleTopicMetaReq().setTopicNameToDrop(topicNameToDrop);
+1
+1 final DataNodeAsyncRequestContext<TPushSingleTopicMetaReq, TPushTopicMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.TOPIC_PUSH_SINGLE_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList().stream()
+1 .map(TPushTopicMetaResp::getStatus)
+1 .collect(Collectors.toList());
+1 }
+1
+1 public Map<Integer, TPushTopicMetaResp> pushMultiTopicMetaToDataNodes(
+1 List<ByteBuffer> topicMetaBinaryList) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushMultiTopicMetaReq request =
+1 new TPushMultiTopicMetaReq().setTopicMetas(topicMetaBinaryList);
+1
+1 final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushTopicMetaResp> dropMultiTopicOnDataNodes(List<String> topicNamesToDrop) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushMultiTopicMetaReq request =
+1 new TPushMultiTopicMetaReq().setTopicNamesToDrop(topicNamesToDrop);
+1
+1 final DataNodeAsyncRequestContext<TPushMultiTopicMetaReq, TPushTopicMetaResp> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.TOPIC_PUSH_MULTI_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public Map<Integer, TPushConsumerGroupMetaResp> pushAllConsumerGroupMetaToDataNodes(
+1 List<ByteBuffer> consumerGroupMetaBinaryList) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushConsumerGroupMetaReq request =
+1 new TPushConsumerGroupMetaReq().setConsumerGroupMetas(consumerGroupMetaBinaryList);
+1
+1 final DataNodeAsyncRequestContext<TPushConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+1 clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_ALL_META, request, dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance()
+1 .sendAsyncRequestToNodeWithRetryAndTimeoutInMs(
+1 clientHandler,
+1 PipeConfig.getInstance().getPipeMetaSyncerSyncIntervalMinutes() * 60 * 1000 * 2 / 3);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 public List<TSStatus> pushSingleConsumerGroupOnDataNode(ByteBuffer consumerGroupMeta) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSingleConsumerGroupMetaReq request =
+1 new TPushSingleConsumerGroupMetaReq().setConsumerGroupMeta(consumerGroupMeta);
+1
+1 final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+1 clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
+1 request,
+1 dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList().stream()
+1 .map(TPushConsumerGroupMetaResp::getStatus)
+1 .collect(Collectors.toList());
+1 }
+1
+1 public List<TSStatus> dropSingleConsumerGroupOnDataNode(String consumerGroupNameToDrop) {
+1 final Map<Integer, TDataNodeLocation> dataNodeLocationMap =
+1 configManager.getNodeManager().getRegisteredDataNodeLocations();
+1 final TPushSingleConsumerGroupMetaReq request =
+1 new TPushSingleConsumerGroupMetaReq().setConsumerGroupNameToDrop(consumerGroupNameToDrop);
+1
+1 final DataNodeAsyncRequestContext<TPushSingleConsumerGroupMetaReq, TPushConsumerGroupMetaResp>
+1 clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.CONSUMER_GROUP_PUSH_SINGLE_META,
+1 request,
+1 dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseList().stream()
+1 .map(TPushConsumerGroupMetaResp::getStatus)
+1 .collect(Collectors.toList());
+1 }
+1
+1 public LockQueue getNodeLock() {
+1 return nodeLock;
+1 }
+1
+1 public ProcedureScheduler getScheduler() {
+1 return scheduler;
+1 }
+1
+1 public ReentrantLock getSchedulerLock() {
+1 return schedulerLock;
+1 }
+1
+1 public ReentrantLock getSubmitRegionMigrateLock() {
+1 return submitRegionMigrateLock;
+1 }
+1
+1 public RegionMaintainHandler getRegionMaintainHandler() {
+1 return regionMaintainHandler;
+1 }
+1
+1 public RemoveDataNodeHandler getRemoveDataNodeHandler() {
+1 return removeDataNodeHandler;
+1 }
+1
+1 private ConsensusManager getConsensusManager() {
+1 return configManager.getConsensusManager();
+1 }
+1
+1 private NodeManager getNodeManager() {
+1 return configManager.getNodeManager();
+1 }
+1
+1 private ClusterSchemaManager getClusterSchemaManager() {
+1 return configManager.getClusterSchemaManager();
+1 }
+1
+1 private PartitionManager getPartitionManager() {
+1 return configManager.getPartitionManager();
+1 }
+1
+1 private LoadManager getLoadManager() {
+1 return configManager.getLoadManager();
+1 }
+1}
+1
\ No newline at end of file
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
index 928f3cfcd37..ebea1555bc1 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
@@ -1,587 +1,588 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.confignode.procedure.env;
-
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
-import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
-import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
-import org.apache.iotdb.common.rpc.thrift.TEndPoint;
-import org.apache.iotdb.common.rpc.thrift.TRegionMaintainTaskStatus;
-import org.apache.iotdb.common.rpc.thrift.TRegionMigrateFailedType;
-import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
-import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.client.ClientPoolFactory;
-import org.apache.iotdb.commons.client.IClientManager;
-import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.RegionStatus;
-import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
-import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
-import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
-import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
-import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
-import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
-import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
-import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
-import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan;
-import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan;
-import org.apache.iotdb.confignode.manager.ConfigManager;
-import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
-import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
-import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq;
-import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq;
-import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp;
-import org.apache.iotdb.mpp.rpc.thrift.TRegionMigrateResult;
-import org.apache.iotdb.mpp.rpc.thrift.TResetPeerListReq;
-import org.apache.iotdb.rpc.TSStatusCode;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REGION_MIGRATE_PROCESS;
-import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS;
-import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS_V2;
-import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS;
-
-public class RegionMaintainHandler {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(RegionMaintainHandler.class);
-
- private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
-
- private final ConfigManager configManager;
-
- private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> dataNodeClientManager;
-
- public RegionMaintainHandler(ConfigManager configManager) {
- this.configManager = configManager;
- dataNodeClientManager =
- new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
- .createClientManager(
- new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
- }
-
- public static String getIdWithRpcEndpoint(TDataNodeLocation location) {
- return String.format(
- "[dataNodeId: %s, clientRpcEndPoint: %s]",
- location.getDataNodeId(), location.getClientRpcEndPoint());
- }
-
- public static String simplifiedLocation(TDataNodeLocation dataNodeLocation) {
- return dataNodeLocation.getDataNodeId() + "@" + dataNodeLocation.getInternalEndPoint().getIp();
- }
-
- /**
- * Find dest data node.
- *
- * @param regionId region id
- * @return dest data node location
- */
- public TDataNodeLocation findDestDataNode(TConsensusGroupId regionId) {
- TSStatus status;
- List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
- if (regionReplicaNodes.isEmpty()) {
- LOGGER.warn("Cannot find region replica nodes, region: {}", regionId);
- status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage("Cannot find region replica nodes, region: " + regionId);
- return null;
- }
-
- Optional<TDataNodeLocation> newNode = pickNewReplicaNodeForRegion(regionReplicaNodes);
- if (!newNode.isPresent()) {
- LOGGER.warn("No enough Data node to migrate region: {}", regionId);
- return null;
- }
- return newNode.get();
- }
-
- /**
- * Create a new RegionReplica and build the ConsensusGroup on the destined DataNode
- *
- * <p>createNewRegionPeer should be invoked on a DataNode that doesn't contain any peer of the
- * specific ConsensusGroup, in order to avoid there exists one DataNode who has more than one
- * RegionReplica.
- *
- * @param regionId The given ConsensusGroup
- * @param destDataNode The destined DataNode where the new peer will be created
- * @return status
- */
- public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocation destDataNode) {
- TSStatus status;
- List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
- if (regionReplicaNodes.isEmpty()) {
- LOGGER.warn(
- "{}, Cannot find region replica nodes in createPeer, regionId: {}",
- REGION_MIGRATE_PROCESS,
- regionId);
- status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
- status.setMessage("Not find region replica nodes in createPeer, regionId: " + regionId);
- return status;
- }
-
- List<TDataNodeLocation> currentPeerNodes;
- if (TConsensusGroupType.DataRegion.equals(regionId.getType())
- && (IOT_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())
- || IOT_CONSENSUS_V2.equals(CONF.getDataRegionConsensusProtocolClass()))) {
- // parameter of createPeer for MultiLeader should be all peers
- currentPeerNodes = new ArrayList<>(regionReplicaNodes);
- currentPeerNodes.add(destDataNode);
- } else {
- // parameter of createPeer for Ratis can be empty
- currentPeerNodes = Collections.emptyList();
- }
-
- String database = configManager.getPartitionManager().getRegionDatabase(regionId);
- TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, database);
-
- status =
- (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- destDataNode.getInternalEndPoint(),
- req,
- CnToDnSyncRequestType.CREATE_NEW_REGION_PEER);
-
- if (isSucceed(status)) {
- LOGGER.info(
- "{}, Send action createNewRegionPeer finished, regionId: {}, newPeerDataNodeId: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- getIdWithRpcEndpoint(destDataNode));
- } else {
- LOGGER.error(
- "{}, Send action createNewRegionPeer error, regionId: {}, newPeerDataNodeId: {}, result: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- getIdWithRpcEndpoint(destDataNode),
- status);
- }
-
- return status;
- }
-
- /**
- * Order the specific ConsensusGroup to add peer for the new RegionReplica.
- *
- * <p>The add peer interface could be invoked at any DataNode who contains one of the
- * RegionReplica of the specified ConsensusGroup except the new one
- *
- * @param destDataNode The DataNodeLocation where the new RegionReplica is created
- * @param regionId region id
- * @return TSStatus
- */
- public TSStatus submitAddRegionPeerTask(
- long procedureId,
- TDataNodeLocation destDataNode,
- TConsensusGroupId regionId,
- TDataNodeLocation coordinator) {
- TSStatus status;
-
- // Send addRegionPeer request to the selected DataNode,
- // destDataNode is where the new RegionReplica is created
- TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId);
- status =
- (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- coordinator.getInternalEndPoint(),
- maintainPeerReq,
- CnToDnSyncRequestType.ADD_REGION_PEER);
- LOGGER.info(
- "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- getIdWithRpcEndpoint(coordinator),
- getIdWithRpcEndpoint(destDataNode),
- status);
- return status;
- }
-
- /**
- * Order the specific ConsensusGroup to remove peer for the old RegionReplica.
- *
- * <p>The remove peer interface could be invoked at any DataNode who contains one of the
- * RegionReplica of the specified ConsensusGroup except the origin one
- *
- * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
- * @param regionId region id
- * @return TSStatus
- */
- public TSStatus submitRemoveRegionPeerTask(
- long procedureId,
- TDataNodeLocation originalDataNode,
- TConsensusGroupId regionId,
- TDataNodeLocation coordinator) {
- TSStatus status;
-
- // Send removeRegionPeer request to the rpcClientDataNode
- TMaintainPeerReq maintainPeerReq =
- new TMaintainPeerReq(regionId, originalDataNode, procedureId);
- status =
- (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- coordinator.getInternalEndPoint(),
- maintainPeerReq,
- CnToDnSyncRequestType.REMOVE_REGION_PEER);
- LOGGER.info(
- "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- getIdWithRpcEndpoint(coordinator));
- return status;
- }
-
- /**
- * Delete a Region peer in the given ConsensusGroup and all of its data on the specified DataNode
- *
- * <p>If the originalDataNode is down, we should delete local data and do other cleanup works
- * manually.
- *
- * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
- * @param regionId region id
- * @return TSStatus
- */
- public TSStatus submitDeleteOldRegionPeerTask(
- long procedureId, TDataNodeLocation originalDataNode, TConsensusGroupId regionId) {
-
- TSStatus status;
- TMaintainPeerReq maintainPeerReq =
- new TMaintainPeerReq(regionId, originalDataNode, procedureId);
-
- status =
- configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId())
- == NodeStatus.Unknown
- ? (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithGivenRetry(
- originalDataNode.getInternalEndPoint(),
- maintainPeerReq,
- CnToDnSyncRequestType.DELETE_OLD_REGION_PEER,
- 1)
- : (TSStatus)
- SyncDataNodeClientPool.getInstance()
- .sendSyncRequestToDataNodeWithRetry(
- originalDataNode.getInternalEndPoint(),
- maintainPeerReq,
- CnToDnSyncRequestType.DELETE_OLD_REGION_PEER);
- LOGGER.info(
- "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- originalDataNode.getInternalEndPoint());
- return status;
- }
-
- public Map<Integer, TSStatus> resetPeerList(
- TConsensusGroupId regionId,
- List<TDataNodeLocation> correctDataNodeLocations,
- Map<Integer, TDataNodeLocation> dataNodeLocationMap) {
- DataNodeAsyncRequestContext<TResetPeerListReq, TSStatus> clientHandler =
- new DataNodeAsyncRequestContext<>(
- CnToDnAsyncRequestType.RESET_PEER_LIST,
- new TResetPeerListReq(regionId, correctDataNodeLocations),
- dataNodeLocationMap);
- CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
- return clientHandler.getResponseMap();
- }
-
- // TODO: will use 'procedure yield' to refactor later
- public TRegionMigrateResult waitTaskFinish(long taskId, TDataNodeLocation dataNodeLocation) {
- final long MAX_DISCONNECTION_TOLERATE_MS = 600_000;
- final long INITIAL_DISCONNECTION_TOLERATE_MS = 60_000;
- long startTime = System.nanoTime();
- long lastReportTime = System.nanoTime();
- while (true) {
- try (SyncDataNodeInternalServiceClient dataNodeClient =
- dataNodeClientManager.borrowClient(dataNodeLocation.getInternalEndPoint())) {
- TRegionMigrateResult report = dataNodeClient.getRegionMaintainResult(taskId);
- lastReportTime = System.nanoTime();
- if (report.getTaskStatus() != TRegionMaintainTaskStatus.PROCESSING) {
- return report;
- }
- } catch (Exception ignore) {
-
- }
- long waitTime =
- Math.min(
- INITIAL_DISCONNECTION_TOLERATE_MS
- + TimeUnit.NANOSECONDS.toMillis(lastReportTime - startTime) / 60,
- MAX_DISCONNECTION_TOLERATE_MS);
- long disconnectionTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime);
- if (disconnectionTime > waitTime) {
- LOGGER.warn(
- "{} task {} cannot get task report from DataNode {}, last report time is {} ago",
- REGION_MIGRATE_PROCESS,
- taskId,
- dataNodeLocation,
- CommonDateTimeUtils.convertMillisecondToDurationStr(
- TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime)));
- TRegionMigrateResult report = new TRegionMigrateResult();
- report.setTaskStatus(TRegionMaintainTaskStatus.FAIL);
- report.setFailedNodeAndReason(new HashMap<>());
- report.getFailedNodeAndReason().put(dataNodeLocation, TRegionMigrateFailedType.Disconnect);
- return report;
- }
- try {
- TimeUnit.SECONDS.sleep(1);
- } catch (InterruptedException ignore) {
- Thread.currentThread().interrupt();
- return new TRegionMigrateResult(TRegionMaintainTaskStatus.PROCESSING);
- }
- }
- }
-
- public void addRegionLocation(TConsensusGroupId regionId, TDataNodeLocation newLocation) {
- AddRegionLocationPlan req = new AddRegionLocationPlan(regionId, newLocation);
- TSStatus status = configManager.getPartitionManager().addRegionLocation(req);
- LOGGER.info(
- "AddRegionLocation finished, add region {} to {}, result is {}",
- regionId,
- getIdWithRpcEndpoint(newLocation),
- status);
- configManager
- .getLoadManager()
- .getLoadCache()
- .createRegionCache(regionId, newLocation.getDataNodeId());
- }
-
- public void forceUpdateRegionCache(
- TConsensusGroupId regionId, TDataNodeLocation newLocation, RegionStatus regionStatus) {
- configManager
- .getLoadManager()
- .forceUpdateRegionCache(regionId, newLocation.getDataNodeId(), regionStatus);
- }
-
- public void removeRegionLocation(
- TConsensusGroupId regionId, TDataNodeLocation deprecatedLocation) {
- RemoveRegionLocationPlan req = new RemoveRegionLocationPlan(regionId, deprecatedLocation);
- TSStatus status = configManager.getPartitionManager().removeRegionLocation(req);
- LOGGER.info(
- "RemoveRegionLocation remove region {} from DataNode {}, result is {}",
- regionId,
- getIdWithRpcEndpoint(deprecatedLocation),
- status);
- configManager.getLoadManager().removeRegionCache(regionId, deprecatedLocation.getDataNodeId());
- configManager.getLoadManager().getRouteBalancer().balanceRegionLeaderAndPriority();
- }
-
- /**
- * Find all DataNodes which contains the given regionId
- *
- * @param regionId region id
- * @return DataNode locations
- */
- public List<TDataNodeLocation> findRegionLocations(TConsensusGroupId regionId) {
- Optional<TRegionReplicaSet> regionReplicaSet = getRegionReplicaSet(regionId);
- if (regionReplicaSet.isPresent()) {
- return regionReplicaSet.get().getDataNodeLocations();
- }
- return Collections.emptyList();
- }
-
- public Optional<TRegionReplicaSet> getRegionReplicaSet(TConsensusGroupId regionId) {
- return configManager.getPartitionManager().getAllReplicaSets().stream()
- .filter(rg -> rg.regionId.equals(regionId))
- .findAny();
- }
-
- public String getRegionReplicaSetString(TConsensusGroupId regionId) {
- Optional<TRegionReplicaSet> regionReplicaSet = getRegionReplicaSet(regionId);
- if (!regionReplicaSet.isPresent()) {
- return "UNKNOWN!";
- }
- StringBuilder result = new StringBuilder(regionReplicaSet.get().getRegionId() + ": {");
- for (TDataNodeLocation dataNodeLocation : regionReplicaSet.get().getDataNodeLocations()) {
- result.append(simplifiedLocation(dataNodeLocation)).append(", ");
- }
- result.append("}");
- return result.toString();
- }
-
- private Optional<TDataNodeLocation> pickNewReplicaNodeForRegion(
- List<TDataNodeLocation> regionReplicaNodes) {
- List<TDataNodeConfiguration> dataNodeConfigurations =
- configManager.getNodeManager().filterDataNodeThroughStatus(NodeStatus.Running);
- // Randomly selected to ensure a basic load balancing
- Collections.shuffle(dataNodeConfigurations);
- return dataNodeConfigurations.stream()
- .map(TDataNodeConfiguration::getLocation)
- .filter(e -> !regionReplicaNodes.contains(e))
- .findAny();
- }
-
- public boolean isSucceed(TSStatus status) {
- return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
- }
-
- public boolean isFailed(TSStatus status) {
- return !isSucceed(status);
- }
-
- /**
- * Change the leader of given Region.
- *
- * <p>For IOT_CONSENSUS, using `changeLeaderForIoTConsensus` method to change the regionLeaderMap
- * maintained in ConfigNode.
- *
- * <p>For RATIS_CONSENSUS, invoking `changeRegionLeader` DataNode RPC method to change the leader.
- *
- * @param regionId The region to be migrated
- * @param originalDataNode The DataNode where the region locates
- */
- public void transferRegionLeader(
- TConsensusGroupId regionId, TDataNodeLocation originalDataNode, TDataNodeLocation coodinator)
- throws ProcedureException, InterruptedException {
- // find new leader
- Optional<TDataNodeLocation> newLeaderNode = Optional.empty();
- List<TDataNodeLocation> excludeDataNode = new ArrayList<>();
- excludeDataNode.add(originalDataNode);
- excludeDataNode.add(coodinator);
- newLeaderNode = filterDataNodeWithOtherRegionReplica(regionId, excludeDataNode);
- if (!newLeaderNode.isPresent()) {
- // If we have no choice, we use it
- newLeaderNode = Optional.of(coodinator);
- }
- // ratis needs DataNode to do election by itself
- long timestamp = System.nanoTime();
- if (TConsensusGroupType.SchemaRegion.equals(regionId.getType())
- || TConsensusGroupType.DataRegion.equals(regionId.getType())
- && RATIS_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())) {
- final int MAX_RETRY_TIME = 10;
- int retryTime = 0;
- long sleepTime =
- (CONF.getSchemaRegionRatisRpcLeaderElectionTimeoutMaxMs()
- + CONF.getSchemaRegionRatisRpcLeaderElectionTimeoutMinMs())
- / 2;
- Integer leaderId = configManager.getLoadManager().getRegionLeaderMap().get(regionId);
-
- if (leaderId != -1) {
- // The migrated node is not leader, so we don't need to transfer temporarily
- if (originalDataNode.getDataNodeId() != leaderId) {
- return;
- }
- }
- while (true) {
- TRegionLeaderChangeResp resp =
- SyncDataNodeClientPool.getInstance()
- .changeRegionLeader(
- regionId, originalDataNode.getInternalEndPoint(), newLeaderNode.get());
- if (resp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- timestamp = resp.getConsensusLogicalTimestamp();
- break;
- }
- if (retryTime++ > MAX_RETRY_TIME) {
- LOGGER.warn("[RemoveRegion] Ratis transfer leader fail, but procedure will continue.");
- return;
- }
- LOGGER.warn(
- "Call changeRegionLeader fail for the {} time, will sleep {} ms", retryTime, sleepTime);
- Thread.sleep(sleepTime);
- }
- }
-
- configManager
- .getLoadManager()
- .forceUpdateConsensusGroupCache(
- Collections.singletonMap(
- regionId,
- new ConsensusGroupHeartbeatSample(timestamp, newLeaderNode.get().getDataNodeId())));
- configManager.getLoadManager().getRouteBalancer().balanceRegionLeaderAndPriority();
-
- LOGGER.info(
- "{}, Change region leader finished, regionId: {}, newLeaderNode: {}",
- REGION_MIGRATE_PROCESS,
- regionId,
- newLeaderNode);
- }
-
- /**
- * Filter a DataNode who contains other RegionReplica excepts the given one.
- *
- * <p>Choose the RUNNING status datanode firstly, if no RUNNING status datanode match the
- * condition, then we choose the REMOVING status datanode.
- *
- * <p>`addRegionPeer`, `removeRegionPeer` and `changeRegionLeader` invoke this method.
- *
- * @param regionId The specific RegionId
- * @param filterLocation The DataNodeLocation that should be filtered
- * @return A DataNodeLocation that contains other RegionReplica and different from the
- * filterLocation
- */
- public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
- TConsensusGroupId regionId, TDataNodeLocation filterLocation) {
- List<TDataNodeLocation> filterLocations = Collections.singletonList(filterLocation);
- return filterDataNodeWithOtherRegionReplica(regionId, filterLocations);
- }
-
- public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
- TConsensusGroupId regionId, List<TDataNodeLocation> filterLocations) {
- return filterDataNodeWithOtherRegionReplica(
- regionId, filterLocations, NodeStatus.Running, NodeStatus.ReadOnly);
- }
-
- public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
- TConsensusGroupId regionId, TDataNodeLocation filterLocation, NodeStatus... allowingStatus) {
- List<TDataNodeLocation> excludeLocations = Collections.singletonList(filterLocation);
- return filterDataNodeWithOtherRegionReplica(regionId, excludeLocations, allowingStatus);
- }
-
- public Optional<TDataNodeLocation> filterDataNodeWithOtherRegionReplica(
- TConsensusGroupId regionId,
- List<TDataNodeLocation> excludeLocations,
- NodeStatus... allowingStatus) {
- List<TDataNodeLocation> regionLocations = findRegionLocations(regionId);
- if (regionLocations.isEmpty()) {
- LOGGER.warn("Cannot find DataNodes contain the given region: {}", regionId);
- return Optional.empty();
- }
-
- // Choosing the RUNNING DataNodes to execute firstly
- // If all DataNodes are not RUNNING, then choose the REMOVING DataNodes secondly
- List<TDataNodeLocation> aliveDataNodes =
- configManager.getNodeManager().filterDataNodeThroughStatus(allowingStatus).stream()
- .map(TDataNodeConfiguration::getLocation)
- .collect(Collectors.toList());
- final int leaderId = configManager.getLoadManager().getRegionLeaderMap().get(regionId);
- Collections.shuffle(aliveDataNodes);
- Optional<TDataNodeLocation> bestChoice = Optional.empty();
- for (TDataNodeLocation aliveDataNode : aliveDataNodes) {
- if (regionLocations.contains(aliveDataNode) && !excludeLocations.contains(aliveDataNode)) {
- if (leaderId == aliveDataNode.getDataNodeId()) {
- bestChoice = Optional.of(aliveDataNode);
- break;
- } else if (!bestChoice.isPresent()) {
- bestChoice = Optional.of(aliveDataNode);
- }
- }
- }
- return bestChoice;
- }
-}
+1/*
+1 * Licensed to the Apache Software Foundation (ASF) under one
+1 * or more contributor license agreements. See the NOTICE file
+1 * distributed with this work for additional information
+1 * regarding copyright ownership. The ASF licenses this file
+1 * to you under the Apache License, Version 2.0 (the
+1 * "License"); you may not use this file except in compliance
+1 * with the License. You may obtain a copy of the License at
+1 *
+1 * http://www.apache.org/licenses/LICENSE-2.0
+1 *
+1 * Unless required by applicable law or agreed to in writing,
+1 * software distributed under the License is distributed on an
+1 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+1 * KIND, either express or implied. See the License for the
+1 * specific language governing permissions and limitations
+1 * under the License.
+1 */
+1package org.apache.iotdb.confignode.procedure.env;
+1
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+1import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+1import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+1import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+1import org.apache.iotdb.common.rpc.thrift.TRegionMaintainTaskStatus;
+1import org.apache.iotdb.common.rpc.thrift.TRegionMigrateFailedType;
+1import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+1import org.apache.iotdb.common.rpc.thrift.TSStatus;
+1import org.apache.iotdb.commons.client.ClientPoolFactory;
+1import org.apache.iotdb.commons.client.IClientManager;
+1import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+1import org.apache.iotdb.commons.cluster.NodeStatus;
+1import org.apache.iotdb.commons.cluster.RegionStatus;
+1import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
+1import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+1import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+1import org.apache.iotdb.confignode.client.async.handlers.DataNodeAsyncRequestContext;
+1import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
+1import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
+1import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+1import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+1import org.apache.iotdb.confignode.consensus.request.write.partition.AddRegionLocationPlan;
+1import org.apache.iotdb.confignode.consensus.request.write.partition.RemoveRegionLocationPlan;
+1import org.apache.iotdb.confignode.manager.ConfigManager;
+1import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
+1import org.apache.iotdb.confignode.procedure.exception.ProcedureException;
+1import org.apache.iotdb.mpp.rpc.thrift.TCreatePeerReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TMaintainPeerReq;
+1import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp;
+1import org.apache.iotdb.mpp.rpc.thrift.TRegionMigrateResult;
+1import org.apache.iotdb.mpp.rpc.thrift.TResetPeerListReq;
+1import org.apache.iotdb.rpc.TSStatusCode;
+1
+1import org.slf4j.Logger;
+1import org.slf4j.LoggerFactory;
+1
+1import java.util.ArrayList;
+1import java.util.Collections;
+1import java.util.HashMap;
+1import java.util.List;
+1import java.util.Map;
+1import java.util.Optional;
+1import java.util.concurrent.TimeUnit;
+1import java.util.stream.Collectors;
+1
+1import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.REGION_MIGRATE_PROCESS;
+1import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS;
+1import static org.apache.iotdb.consensus.ConsensusFactory.IOT_CONSENSUS_V2;
+1import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS;
+1
+1public class RegionMaintainHandler {
+1
+1 private static final Logger LOGGER = LoggerFactory.getLogger(RegionMaintainHandler.class);
+1
+1 private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
+1
+1 private final ConfigManager configManager;
+1
+1 private final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> dataNodeClientManager;
+1
+1 public RegionMaintainHandler(ConfigManager configManager) {
+1 this.configManager = configManager;
+1 dataNodeClientManager =
+1 new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
+1 .createClientManager(
+1 new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+1 }
+1
+1 public static String getIdWithRpcEndpoint(TDataNodeLocation location) {
+1 return String.format(
+1 "[dataNodeId: %s, clientRpcEndPoint: %s]",
+1 location.getDataNodeId(), location.getClientRpcEndPoint());
+1 }
+1
+1 public static String simplifiedLocation(TDataNodeLocation dataNodeLocation) {
+1 return dataNodeLocation.getDataNodeId() + "@" + dataNodeLocation.getInternalEndPoint().getIp();
+1 }
+1
+1 /**
+1 * Find dest data node.
+1 *
+1 * @param regionId region id
+1 * @return dest data node location
+1 */
+1 public TDataNodeLocation findDestDataNode(TConsensusGroupId regionId) {
+1 TSStatus status;
+1 List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
+1 if (regionReplicaNodes.isEmpty()) {
+1 LOGGER.warn("Cannot find region replica nodes, region: {}", regionId);
+1 status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+1 status.setMessage("Cannot find region replica nodes, region: " + regionId);
+1 return null;
+1 }
+1
+1 Optional<TDataNodeLocation> newNode = pickNewReplicaNodeForRegion(regionReplicaNodes);
+1 if (!newNode.isPresent()) {
+1 LOGGER.warn("No enough Data node to migrate region: {}", regionId);
+1 return null;
+1 }
+1 return newNode.get();
+1 }
+1
+1 /**
+1 * Create a new RegionReplica and build the ConsensusGroup on the destined DataNode
+1 *
+1 * <p>createNewRegionPeer should be invoked on a DataNode that doesn't contain any peer of the
+1 * specific ConsensusGroup, in order to avoid there exists one DataNode who has more than one
+1 * RegionReplica.
+1 *
+1 * @param regionId The given ConsensusGroup
+1 * @param destDataNode The destined DataNode where the new peer will be created
+1 * @return status
+1 */
+1 public TSStatus createNewRegionPeer(TConsensusGroupId regionId, TDataNodeLocation destDataNode) {
+1 TSStatus status;
+1 List<TDataNodeLocation> regionReplicaNodes = findRegionLocations(regionId);
+1 if (regionReplicaNodes.isEmpty()) {
+1 LOGGER.warn(
+1 "{}, Cannot find region replica nodes in createPeer, regionId: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId);
+1 status = new TSStatus(TSStatusCode.MIGRATE_REGION_ERROR.getStatusCode());
+1 status.setMessage("Not find region replica nodes in createPeer, regionId: " + regionId);
+1 return status;
+1 }
+1
+1 List<TDataNodeLocation> currentPeerNodes;
+1 if (TConsensusGroupType.DataRegion.equals(regionId.getType())
+1 && (IOT_CONSENSUS.equals(CONF.getDataRegionConsensusProtocolClass())
+1 || IOT_CONSENSUS_V2.equals(CONF.getDataRegionConsensusProtocolClass()))) {
+1 // parameter of createPeer for MultiLeader should be all peers
+1 currentPeerNodes = new ArrayList<>(regionReplicaNodes);
+1 currentPeerNodes.add(destDataNode);
+1 } else {
+1 // parameter of createPeer for Ratis can be empty
+1 currentPeerNodes = Collections.emptyList();
+1 }
+1
+1 String database = configManager.getPartitionManager().getRegionDatabase(regionId);
+1 TCreatePeerReq req = new TCreatePeerReq(regionId, currentPeerNodes, database);
+1
+1 status =
+1 (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 destDataNode.getInternalEndPoint(),
+1 req,
+1 CnToDnSyncRequestType.CREATE_NEW_REGION_PEER);
+1
+1 if (isSucceed(status)) {
+1 LOGGER.info(
+1 "{}, Send action createNewRegionPeer finished, regionId: {}, newPeerDataNodeId: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId,
+1 getIdWithRpcEndpoint(destDataNode));
+1 } else {
+1 LOGGER.error(
+1 "{}, Send action createNewRegionPeer error, regionId: {}, newPeerDataNodeId: {}, result: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId,
+1 getIdWithRpcEndpoint(destDataNode),
+1 status);
+1 }
+1
+1 return status;
+1 }
+1
+1 /**
+1 * Order the specific ConsensusGroup to add peer for the new RegionReplica.
+1 *
+1 * <p>The add peer interface could be invoked at any DataNode who contains one of the
+1 * RegionReplica of the specified ConsensusGroup except the new one
+1 *
+1 * @param destDataNode The DataNodeLocation where the new RegionReplica is created
+1 * @param regionId region id
+1 * @return TSStatus
+1 */
+1 public TSStatus submitAddRegionPeerTask(
+1 long procedureId,
+1 TDataNodeLocation destDataNode,
+1 TConsensusGroupId regionId,
+1 TDataNodeLocation coordinator) {
+1 TSStatus status;
+1
+1 // Send addRegionPeer request to the selected DataNode,
+1 // destDataNode is where the new RegionReplica is created
+1 TMaintainPeerReq maintainPeerReq = new TMaintainPeerReq(regionId, destDataNode, procedureId);
+1 status =
+1 (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 coordinator.getInternalEndPoint(),
+1 maintainPeerReq,
+1 CnToDnSyncRequestType.ADD_REGION_PEER);
+1 LOGGER.info(
+1 "{}, Send action addRegionPeer finished, regionId: {}, rpcDataNode: {}, destDataNode: {}, status: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId,
+1 getIdWithRpcEndpoint(coordinator),
+1 getIdWithRpcEndpoint(destDataNode),
+1 status);
+1 return status;
+1 }
+1
+1 /**
+1 * Order the specific ConsensusGroup to remove peer for the old RegionReplica.
+1 *
+1 * <p>The remove peer interface could be invoked at any DataNode who contains one of the
+1 * RegionReplica of the specified ConsensusGroup except the origin one
+1 *
+1 * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
+1 * @param regionId region id
+1 * @return TSStatus
+1 */
+1 public TSStatus submitRemoveRegionPeerTask(
+1 long procedureId,
+1 TDataNodeLocation originalDataNode,
+1 TConsensusGroupId regionId,
+1 TDataNodeLocation coordinator) {
+1 TSStatus status;
+1
+1 // Send removeRegionPeer request to the rpcClientDataNode
+1 TMaintainPeerReq maintainPeerReq =
+1 new TMaintainPeerReq(regionId, originalDataNode, procedureId);
+1 status =
+1 (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 coordinator.getInternalEndPoint(),
+1 maintainPeerReq,
+1 CnToDnSyncRequestType.REMOVE_REGION_PEER);
+1 LOGGER.info(
+1 "{}, Send action removeRegionPeer finished, regionId: {}, rpcDataNode: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId,
+1 getIdWithRpcEndpoint(coordinator));
+1 return status;
+1 }
+1
+1 /**
+1 * Delete a Region peer in the given ConsensusGroup and all of its data on the specified DataNode
+1 *
+1 * <p>If the originalDataNode is down, we should delete local data and do other cleanup works
+1 * manually.
+1 *
+1 * @param originalDataNode The DataNodeLocation who contains the original RegionReplica
+1 * @param regionId region id
+1 * @return TSStatus
+1 */
+1 public TSStatus submitDeleteOldRegionPeerTask(
+1 long procedureId, TDataNodeLocation originalDataNode, TConsensusGroupId regionId) {
+1
+1 TSStatus status;
+1 TMaintainPeerReq maintainPeerReq =
+1 new TMaintainPeerReq(regionId, originalDataNode, procedureId);
+1
+1 status =
+1 configManager.getLoadManager().getNodeStatus(originalDataNode.getDataNodeId())
+1 == NodeStatus.Unknown
+1 ? (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithGivenRetry(
+1 originalDataNode.getInternalEndPoint(),
+1 maintainPeerReq,
+1 CnToDnSyncRequestType.DELETE_OLD_REGION_PEER,
+1 1)
+1 : (TSStatus)
+1 SyncDataNodeClientPool.getInstance()
+1 .sendSyncRequestToDataNodeWithRetry(
+1 originalDataNode.getInternalEndPoint(),
+1 maintainPeerReq,
+1 CnToDnSyncRequestType.DELETE_OLD_REGION_PEER);
+1 LOGGER.info(
+1 "{}, Send action deleteOldRegionPeer finished, regionId: {}, dataNodeId: {}",
+1 REGION_MIGRATE_PROCESS,
+1 regionId,
+1 originalDataNode.getInternalEndPoint());
+1 return status;
+1 }
+1
+1 public Map<Integer, TSStatus> resetPeerList(
+1 TConsensusGroupId regionId,
+1 List<TDataNodeLocation> correctDataNodeLocations,
+1 Map<Integer, TDataNodeLocation> dataNodeLocationMap) {
+1 DataNodeAsyncRequestContext<TResetPeerListReq, TSStatus> clientHandler =
+1 new DataNodeAsyncRequestContext<>(
+1 CnToDnAsyncRequestType.RESET_PEER_LIST,
+1 new TResetPeerListReq(regionId, correctDataNodeLocations),
+1 dataNodeLocationMap);
+1 CnToDnInternalServiceAsyncRequestManager.getInstance().sendAsyncRequestWithRetry(clientHandler);
+1 return clientHandler.getResponseMap();
+1 }
+1
+1 // TODO: will use 'procedure yield' to refactor later
+1 public TRegionMigrateResult waitTaskFinish(long taskId, TDataNodeLocation dataNodeLocation) {
+1 final long MAX_DISCONNECTION_TOLERATE_MS = 600_000;
+1 final long INITIAL_DISCONNECTION_TOLERATE_MS = 60_000;
+1 long startTime = System.nanoTime();
+1 long lastReportTime = System.nanoTime();
+1 while (true) {
+1 try (SyncDataNodeInternalServiceClient dataNodeClient =
+1 dataNodeClientManager.borrowClient(dataNodeLocation.getInternalEndPoint())) {
+1 TRegionMigrateResult report = dataNodeClient.getRegionMaintainResult(taskId);
+1 lastReportTime = System.nanoTime();
+1 if (report.getTaskStatus() != TRegionMaintainTaskStatus.PROCESSING) {
+1 return report;
+1 }
+1 } catch (Exception ignore) {
+1
+1 }
+1 long waitTime =
+1 Math.min(
+1 INITIAL_DISCONNECTION_TOLERATE_MS
+1 + TimeUnit.NANOSECONDS.toMillis(lastReportTime - startTime) / 60,
+1 MAX_DISCONNECTION_TOLERATE_MS);
+1 long disconnectionTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime);
+1 if (disconnectionTime > waitTime) {
+1 LOGGER.warn(
+1 "{} task {} cannot get task report from DataNode {}, last report time is {} ago",
+1 REGION_MIGRATE_PROCESS,
+1 taskId,
+1 dataNodeLocation,
+1 CommonDateTimeUtils.convertMillisecondToDurationStr(
+1 TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - lastReportTime)));
+1 TRegionMigrateResult report = new TRegionMigrateResult();
+1 report.setTaskStatus(TRegionMaintainTaskStatus.FAIL);
+1 report.setFailedNodeAndReason(new HashMap<>());
+1 report.getFailedNodeAndReason().put(dataNodeLocation, TRegionMigrateFailedType.Disconnect);
+1 return report;
+1 }
+1 try {
+1 TimeUnit.SECONDS.sleep(1);
+1 } catch (InterruptedException ignore) {
+1 Thread.currentThread().interrupt();
+1 return new TRegionMigrateResult(TRegionMaintainTaskStatus.PROCESSING);
+1 }
+1 }
+1 }
+1
+1 public void addRegionLocation(TConsensusGroupId regionId, TDataNodeLocation newLocation) {
+1 AddRegionLocationPlan req = new AddRegionLocationPlan(regionId, newLocation);
+1 TSStatus status = configManager.getPartitionManager().addRegionLocation(req);
+1 LOGGER.info(
+1 "AddRegionLocation finished, add region {} to {}, result is {}",
+1 regionId,
+1 getIdWithRpcEndpoint(newLocation),
+1 status);
... 4877 lines suppressed ...