You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2023/09/18 16:52:53 UTC

[iotdb] 01/01: Pipe: fix pipe coordinator deadlock causing CN election timeout

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch fix-pipe-deadlock-1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 47f5e7a8cf48f33682f2865d6e791f8cb80d0657
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue Sep 19 00:52:44 2023 +0800

    Pipe: fix pipe coordinator deadlock causing CN election timeout
---
 .../manager/pipe/runtime/PipeHeartbeatParser.java  |   8 +-
 .../manager/pipe/runtime/PipeMetaSyncer.java       |  12 +-
 .../manager/pipe/task/PipeTaskCoordinator.java     |  28 ++--
 .../manager/pipe/task/PipeTaskCoordinatorLock.java |  30 ++++
 .../impl/pipe/AbstractOperatePipeProcedureV2.java  | 176 +++++++++++++++------
 5 files changed, 181 insertions(+), 73 deletions(-)

diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
index fe39e9b397a..f6ffb55860f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
@@ -101,7 +101,13 @@ public class PipeHeartbeatParser {
         .submit(
             () -> {
               final AtomicReference<PipeTaskInfo> pipeTaskInfo =
-                  configManager.getPipeManager().getPipeTaskCoordinator().lock();
+                  configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+              if (pipeTaskInfo == null) {
+                LOGGER.warn(
+                    "Failed to acquire lock when parseHeartbeat from data node (id={}).",
+                    dataNodeId);
+                return;
+              }
 
               try {
                 if (!pipeMetaByteBufferListFromDataNode.isEmpty()) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
index 745c7ea299e..82482aa8c41 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
@@ -121,7 +121,11 @@ public class PipeMetaSyncer {
 
   private boolean autoRestartWithLock() {
     final AtomicReference<PipeTaskInfo> pipeTaskInfo =
-        configManager.getPipeManager().getPipeTaskCoordinator().lock();
+        configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+    if (pipeTaskInfo == null) {
+      LOGGER.warn("Failed to acquire pipe lock for auto restart pipe task.");
+      return false;
+    }
     try {
       return pipeTaskInfo.get().autoRestart();
     } finally {
@@ -131,7 +135,11 @@ public class PipeMetaSyncer {
 
   private void handleSuccessfulRestartWithLock() {
     final AtomicReference<PipeTaskInfo> pipeTaskInfo =
-        configManager.getPipeManager().getPipeTaskCoordinator().lock();
+        configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+    if (pipeTaskInfo == null) {
+      LOGGER.warn("Failed to acquire pipe lock for handling successful restart.");
+      return;
+    }
     try {
       pipeTaskInfo.get().handleSuccessfulRestart();
     } finally {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
index 5bec78d66b9..2e378bb9b7b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
@@ -63,10 +63,11 @@ public class PipeTaskCoordinator {
    * @return the pipe task info holder, which can be used to get the pipe task info. The holder is
    *     null if the lock is not acquired.
    */
-  public AtomicReference<PipeTaskInfo> lock() {
-    pipeTaskCoordinatorLock.lock();
+  public AtomicReference<PipeTaskInfo> tryLock() {
+    if (pipeTaskCoordinatorLock.tryLock()) {
+      pipeTaskInfoHolder = new AtomicReference<>(pipeTaskInfo);
+    }
 
-    pipeTaskInfoHolder = new AtomicReference<>(pipeTaskInfo);
     return pipeTaskInfoHolder;
   }
 
@@ -92,12 +93,12 @@ public class PipeTaskCoordinator {
     }
   }
 
-  /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+  /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
   public TSStatus createPipe(TCreatePipeReq req) {
     return configManager.getProcedureManager().createPipe(req);
   }
 
-  /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+  /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
   public TSStatus startPipe(String pipeName) {
     final boolean hasException = pipeTaskInfo.hasExceptions(pipeName);
     final TSStatus status = configManager.getProcedureManager().startPipe(pipeName);
@@ -108,7 +109,7 @@ public class PipeTaskCoordinator {
     return status;
   }
 
-  /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+  /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
   public TSStatus stopPipe(String pipeName) {
     final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName);
     final TSStatus status = configManager.getProcedureManager().stopPipe(pipeName);
@@ -121,7 +122,7 @@ public class PipeTaskCoordinator {
     return status;
   }
 
-  /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+  /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
   public TSStatus dropPipe(String pipeName) {
     final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName);
     final TSStatus status = configManager.getProcedureManager().dropPipe(pipeName);
@@ -137,7 +138,6 @@ public class PipeTaskCoordinator {
   }
 
   public TShowPipeResp showPipes(TShowPipeReq req) {
-    lock();
     try {
       return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2()))
           .filter(req.whereClause, req.pipeName)
@@ -147,13 +147,10 @@ public class PipeTaskCoordinator {
       TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
       res.setMessage(e.getMessage());
       return new PipeTableResp(res, Collections.emptyList()).convertToTShowPipeResp();
-    } finally {
-      unlock();
     }
   }
 
   public TGetAllPipeInfoResp getAllPipeInfo() {
-    lock();
     try {
       return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2()))
           .convertToTGetAllPipeInfoResp();
@@ -162,17 +159,10 @@ public class PipeTaskCoordinator {
       return new TGetAllPipeInfoResp(
           new TSStatus(TSStatusCode.PIPE_ERROR.getStatusCode()).setMessage(e.getMessage()),
           Collections.emptyList());
-    } finally {
-      unlock();
     }
   }
 
   public boolean hasAnyPipe() {
-    lock();
-    try {
-      return !pipeTaskInfo.isEmpty();
-    } finally {
-      unlock();
-    }
+    return !pipeTaskInfo.isEmpty();
   }
 }
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
index 78c1f28ba21..fff9e6e1029 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
 
 import java.util.concurrent.BlockingDeque;
 import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
@@ -57,6 +58,35 @@ public class PipeTaskCoordinatorLock {
     }
   }
 
+  boolean tryLock() {
+    try {
+      final long id = idGenerator.incrementAndGet();
+      LOGGER.info(
+          "PipeTaskCoordinator lock (id: {}) waiting for thread {}",
+          id,
+          Thread.currentThread().getName());
+      if (deque.offer(id, 10, TimeUnit.SECONDS)) {
+        LOGGER.info(
+            "PipeTaskCoordinator lock (id: {}) acquired by thread {}",
+            id,
+            Thread.currentThread().getName());
+        return true;
+      } else {
+        LOGGER.info(
+            "PipeTaskCoordinator lock (id: {}) failed to acquire by thread {} because of timeout",
+            id,
+            Thread.currentThread().getName());
+        return false;
+      }
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      LOGGER.error(
+          "Interrupted while waiting for PipeTaskCoordinator lock, current thread: {}",
+          Thread.currentThread().getName());
+      return false;
+    }
+  }
+
   void unlock() {
     final Long id = deque.poll();
     if (id == null) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
index 892859fd57e..07196c23e1b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
@@ -71,45 +71,78 @@ public abstract class AbstractOperatePipeProcedureV2
 
   @Override
   protected ProcedureLockState acquireLock(ConfigNodeProcedureEnv configNodeProcedureEnv) {
-    configNodeProcedureEnv.getSchedulerLock().lock();
-    try {
-      if (configNodeProcedureEnv.getNodeLock().tryLock(this)) {
-        pipeTaskInfo =
-            configNodeProcedureEnv
-                .getConfigManager()
-                .getPipeManager()
-                .getPipeTaskCoordinator()
-                .lock();
-        LOGGER.info("ProcedureId {} acquire lock.", getProcId());
-        return ProcedureLockState.LOCK_ACQUIRED;
-      }
-      configNodeProcedureEnv.getNodeLock().waitProcedure(this);
-      LOGGER.info("ProcedureId {} wait for lock.", getProcId());
-      return ProcedureLockState.LOCK_EVENT_WAIT;
-    } finally {
-      configNodeProcedureEnv.getSchedulerLock().unlock();
+    LOGGER.info("ProcedureId {} try to acquire pipe lock.", getProcId());
+    pipeTaskInfo =
+        configNodeProcedureEnv
+            .getConfigManager()
+            .getPipeManager()
+            .getPipeTaskCoordinator()
+            .tryLock();
+    if (pipeTaskInfo == null) {
+      LOGGER.warn("ProcedureId {} failed to acquire pipe lock.", getProcId());
+    } else {
+      LOGGER.info("ProcedureId {} acquired pipe lock.", getProcId());
+    }
+
+    final ProcedureLockState procedureLockState = super.acquireLock(configNodeProcedureEnv);
+    switch (procedureLockState) {
+      case LOCK_ACQUIRED:
+        if (pipeTaskInfo == null) {
+          LOGGER.warn(
+              "ProcedureId {}: LOCK_ACQUIRED. The following procedure should not be executed without pipe lock.",
+              getProcId());
+        } else {
+          LOGGER.info(
+              "ProcedureId {}: LOCK_ACQUIRED. The following procedure should be executed with pipe lock.",
+              getProcId());
+        }
+        break;
+      case LOCK_EVENT_WAIT:
+        if (pipeTaskInfo == null) {
+          LOGGER.warn("ProcedureId {}: LOCK_EVENT_WAIT. Without acquiring pipe lock.", getProcId());
+        } else {
+          LOGGER.info("ProcedureId {}: LOCK_EVENT_WAIT. Pipe lock will be released.", getProcId());
+          configNodeProcedureEnv
+              .getConfigManager()
+              .getPipeManager()
+              .getPipeTaskCoordinator()
+              .unlock();
+          pipeTaskInfo = null;
+        }
+        break;
+      default:
+        if (pipeTaskInfo == null) {
+          LOGGER.error(
+              "ProcedureId {}: {}. Invalid lock state. Without acquiring pipe lock.",
+              getProcId(),
+              procedureLockState);
+        } else {
+          LOGGER.error(
+              "ProcedureId {}: {}. Invalid lock state. Pipe lock will be released.",
+              getProcId(),
+              procedureLockState);
+          configNodeProcedureEnv
+              .getConfigManager()
+              .getPipeManager()
+              .getPipeTaskCoordinator()
+              .unlock();
+          pipeTaskInfo = null;
+        }
+        break;
     }
+    return procedureLockState;
   }
 
   @Override
   protected void releaseLock(ConfigNodeProcedureEnv configNodeProcedureEnv) {
-    configNodeProcedureEnv.getSchedulerLock().lock();
-    try {
-      LOGGER.info("ProcedureId {} release lock.", getProcId());
-      if (pipeTaskInfo != null) {
-        configNodeProcedureEnv
-            .getConfigManager()
-            .getPipeManager()
-            .getPipeTaskCoordinator()
-            .unlock();
-      }
-      if (configNodeProcedureEnv.getNodeLock().releaseLock(this)) {
-        configNodeProcedureEnv
-            .getNodeLock()
-            .wakeWaitingProcedures(configNodeProcedureEnv.getScheduler());
-      }
-    } finally {
-      configNodeProcedureEnv.getSchedulerLock().unlock();
+    super.releaseLock(configNodeProcedureEnv);
+
+    if (pipeTaskInfo == null) {
+      LOGGER.warn("ProcedureId {} release lock. No need to release pipe lock.", getProcId());
+    } else {
+      LOGGER.info("ProcedureId {} release lock. Pipe lock will be released.", getProcId());
+      configNodeProcedureEnv.getConfigManager().getPipeManager().getPipeTaskCoordinator().unlock();
+      pipeTaskInfo = null;
     }
   }
 
@@ -145,6 +178,13 @@ public abstract class AbstractOperatePipeProcedureV2
   @Override
   protected Flow executeFromState(ConfigNodeProcedureEnv env, OperatePipeTaskState state)
       throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+    if (pipeTaskInfo == null) {
+      LOGGER.warn(
+          "ProcedureId {}: Pipe lock is not acquired, executeFromState's execution will be skipped.",
+          getProcId());
+      return Flow.NO_MORE_STATE;
+    }
+
     try {
       switch (state) {
         case VALIDATE_TASK:
@@ -170,7 +210,8 @@ public abstract class AbstractOperatePipeProcedureV2
       // Retry before rollback
       if (getCycles() < RETRY_THRESHOLD) {
         LOGGER.warn(
-            "Encountered error when trying to {} at state [{}], retry [{}/{}]",
+            "ProcedureId {}: Encountered error when trying to {} at state [{}], retry [{}/{}]",
+            getProcId(),
             getOperation(),
             state,
             getCycles() + 1,
@@ -180,14 +221,17 @@ public abstract class AbstractOperatePipeProcedureV2
         TimeUnit.MILLISECONDS.sleep(3000L);
       } else {
         LOGGER.warn(
-            "All {} retries failed when trying to {} at state [{}], will rollback...",
+            "ProcedureId {}: All {} retries failed when trying to {} at state [{}], will rollback...",
+            getProcId(),
             RETRY_THRESHOLD,
             getOperation(),
             state,
             e);
         setFailure(
             new ProcedureException(
-                String.format("Fail to %s because %s", getOperation().name(), e.getMessage())));
+                String.format(
+                    "ProcedureId %s: Fail to %s because %s",
+                    getProcId(), getOperation().name(), e.getMessage())));
       }
     }
     return Flow.HAS_MORE_STATE;
@@ -201,29 +245,59 @@ public abstract class AbstractOperatePipeProcedureV2
   @Override
   protected void rollbackState(ConfigNodeProcedureEnv env, OperatePipeTaskState state)
       throws IOException, InterruptedException, ProcedureException {
+    if (pipeTaskInfo == null) {
+      LOGGER.warn(
+          "ProcedureId {}: Pipe lock is not acquired, rollbackState({})'s execution will be skipped.",
+          getProcId(),
+          state);
+      return;
+    }
+
     switch (state) {
       case VALIDATE_TASK:
-        rollbackFromValidateTask(env);
+        try {
+          rollbackFromValidateTask(env);
+        } catch (Exception e) {
+          LOGGER.warn("ProcedureId {}: Failed to rollback from validate task.", getProcId(), e);
+        }
         break;
       case CALCULATE_INFO_FOR_TASK:
-        rollbackFromCalculateInfoForTask(env);
+        try {
+          rollbackFromCalculateInfoForTask(env);
+        } catch (Exception e) {
+          LOGGER.warn(
+              "ProcedureId {}: Failed to rollback from calculate info for task.", getProcId(), e);
+        }
         break;
       case WRITE_CONFIG_NODE_CONSENSUS:
-        // rollbackFromWriteConfigNodeConsensus can be called before rollbackFromOperateOnDataNodes
-        // so we need to check if rollbackFromOperateOnDataNodes is successful executed
-        // if yes, we don't need to call rollbackFromWriteConfigNodeConsensus again
-        if (!isRollbackFromOperateOnDataNodesSuccessful) {
-          rollbackFromWriteConfigNodeConsensus(env);
+        try {
+          // rollbackFromWriteConfigNodeConsensus can be called before
+          // rollbackFromOperateOnDataNodes.
+          // So we need to check if rollbackFromOperateOnDataNodes is successfully executed.
+          // If yes, we don't need to call rollbackFromWriteConfigNodeConsensus again.
+          if (!isRollbackFromOperateOnDataNodesSuccessful) {
+            rollbackFromWriteConfigNodeConsensus(env);
+          }
+        } catch (Exception e) {
+          LOGGER.warn(
+              "ProcedureId {}: Failed to rollback from write config node consensus.",
+              getProcId(),
+              e);
         }
         break;
       case OPERATE_ON_DATA_NODES:
-        // We have to make sure that rollbackFromOperateOnDataNodes is executed before
-        // rollbackFromWriteConfigNodeConsensus, because rollbackFromOperateOnDataNodes is
-        // executed based on the consensus of config nodes that is written by
-        // rollbackFromWriteConfigNodeConsensus
-        rollbackFromWriteConfigNodeConsensus(env);
-        rollbackFromOperateOnDataNodes(env);
-        isRollbackFromOperateOnDataNodesSuccessful = true;
+        try {
+          // We have to make sure that rollbackFromOperateOnDataNodes is executed before
+          // rollbackFromWriteConfigNodeConsensus, because rollbackFromOperateOnDataNodes is
+          // executed based on the consensus of config nodes that is written by
+          // rollbackFromWriteConfigNodeConsensus
+          rollbackFromWriteConfigNodeConsensus(env);
+          rollbackFromOperateOnDataNodes(env);
+          isRollbackFromOperateOnDataNodesSuccessful = true;
+        } catch (Exception e) {
+          LOGGER.warn(
+              "ProcedureId {}: Failed to rollback from operate on data nodes.", getProcId(), e);
+        }
         break;
       default:
         LOGGER.error("Unsupported roll back STATE [{}]", state);