You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2023/09/18 16:52:53 UTC
[iotdb] 01/01: Pipe: fix pipe coordinator deadlock causing CN election timeout
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch fix-pipe-deadlock-1.2
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 47f5e7a8cf48f33682f2865d6e791f8cb80d0657
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Tue Sep 19 00:52:44 2023 +0800
Pipe: fix pipe coordinator deadlock causing CN election timeout
---
.../manager/pipe/runtime/PipeHeartbeatParser.java | 8 +-
.../manager/pipe/runtime/PipeMetaSyncer.java | 12 +-
.../manager/pipe/task/PipeTaskCoordinator.java | 28 ++--
.../manager/pipe/task/PipeTaskCoordinatorLock.java | 30 ++++
.../impl/pipe/AbstractOperatePipeProcedureV2.java | 176 +++++++++++++++------
5 files changed, 181 insertions(+), 73 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
index fe39e9b397a..f6ffb55860f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeHeartbeatParser.java
@@ -101,7 +101,13 @@ public class PipeHeartbeatParser {
.submit(
() -> {
final AtomicReference<PipeTaskInfo> pipeTaskInfo =
- configManager.getPipeManager().getPipeTaskCoordinator().lock();
+ configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+ if (pipeTaskInfo == null) {
+ LOGGER.warn(
+ "Failed to acquire lock when parseHeartbeat from data node (id={}).",
+ dataNodeId);
+ return;
+ }
try {
if (!pipeMetaByteBufferListFromDataNode.isEmpty()) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
index 745c7ea299e..82482aa8c41 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/runtime/PipeMetaSyncer.java
@@ -121,7 +121,11 @@ public class PipeMetaSyncer {
private boolean autoRestartWithLock() {
final AtomicReference<PipeTaskInfo> pipeTaskInfo =
- configManager.getPipeManager().getPipeTaskCoordinator().lock();
+ configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+ if (pipeTaskInfo == null) {
+ LOGGER.warn("Failed to acquire pipe lock for auto restart pipe task.");
+ return false;
+ }
try {
return pipeTaskInfo.get().autoRestart();
} finally {
@@ -131,7 +135,11 @@ public class PipeMetaSyncer {
private void handleSuccessfulRestartWithLock() {
final AtomicReference<PipeTaskInfo> pipeTaskInfo =
- configManager.getPipeManager().getPipeTaskCoordinator().lock();
+ configManager.getPipeManager().getPipeTaskCoordinator().tryLock();
+ if (pipeTaskInfo == null) {
+ LOGGER.warn("Failed to acquire pipe lock for handling successful restart.");
+ return;
+ }
try {
pipeTaskInfo.get().handleSuccessfulRestart();
} finally {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
index 5bec78d66b9..2e378bb9b7b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinator.java
@@ -63,10 +63,11 @@ public class PipeTaskCoordinator {
* @return the pipe task info holder, which can be used to get the pipe task info. The holder is
* null if the lock is not acquired.
*/
- public AtomicReference<PipeTaskInfo> lock() {
- pipeTaskCoordinatorLock.lock();
+ public AtomicReference<PipeTaskInfo> tryLock() {
+ if (pipeTaskCoordinatorLock.tryLock()) {
+ pipeTaskInfoHolder = new AtomicReference<>(pipeTaskInfo);
+ }
- pipeTaskInfoHolder = new AtomicReference<>(pipeTaskInfo);
return pipeTaskInfoHolder;
}
@@ -92,12 +93,12 @@ public class PipeTaskCoordinator {
}
}
- /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+ /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
public TSStatus createPipe(TCreatePipeReq req) {
return configManager.getProcedureManager().createPipe(req);
}
- /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+ /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
public TSStatus startPipe(String pipeName) {
final boolean hasException = pipeTaskInfo.hasExceptions(pipeName);
final TSStatus status = configManager.getProcedureManager().startPipe(pipeName);
@@ -108,7 +109,7 @@ public class PipeTaskCoordinator {
return status;
}
- /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+ /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
public TSStatus stopPipe(String pipeName) {
final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName);
final TSStatus status = configManager.getProcedureManager().stopPipe(pipeName);
@@ -121,7 +122,7 @@ public class PipeTaskCoordinator {
return status;
}
- /** Caller should ensure that the method is called in the lock {@link #lock()}. */
+ /** Caller should ensure that the method is called in the lock {@link #tryLock()}. */
public TSStatus dropPipe(String pipeName) {
final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName);
final TSStatus status = configManager.getProcedureManager().dropPipe(pipeName);
@@ -137,7 +138,6 @@ public class PipeTaskCoordinator {
}
public TShowPipeResp showPipes(TShowPipeReq req) {
- lock();
try {
return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2()))
.filter(req.whereClause, req.pipeName)
@@ -147,13 +147,10 @@ public class PipeTaskCoordinator {
TSStatus res = new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
res.setMessage(e.getMessage());
return new PipeTableResp(res, Collections.emptyList()).convertToTShowPipeResp();
- } finally {
- unlock();
}
}
public TGetAllPipeInfoResp getAllPipeInfo() {
- lock();
try {
return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2()))
.convertToTGetAllPipeInfoResp();
@@ -162,17 +159,10 @@ public class PipeTaskCoordinator {
return new TGetAllPipeInfoResp(
new TSStatus(TSStatusCode.PIPE_ERROR.getStatusCode()).setMessage(e.getMessage()),
Collections.emptyList());
- } finally {
- unlock();
}
}
public boolean hasAnyPipe() {
- lock();
- try {
- return !pipeTaskInfo.isEmpty();
- } finally {
- unlock();
- }
+ return !pipeTaskInfo.isEmpty();
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
index 78c1f28ba21..fff9e6e1029 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/task/PipeTaskCoordinatorLock.java
@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;
import java.util.concurrent.BlockingDeque;
import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
/**
@@ -57,6 +58,35 @@ public class PipeTaskCoordinatorLock {
}
}
+ boolean tryLock() {
+ try {
+ final long id = idGenerator.incrementAndGet();
+ LOGGER.info(
+ "PipeTaskCoordinator lock (id: {}) waiting for thread {}",
+ id,
+ Thread.currentThread().getName());
+ if (deque.offer(id, 10, TimeUnit.SECONDS)) {
+ LOGGER.info(
+ "PipeTaskCoordinator lock (id: {}) acquired by thread {}",
+ id,
+ Thread.currentThread().getName());
+ return true;
+ } else {
+ LOGGER.info(
+ "PipeTaskCoordinator lock (id: {}) failed to acquire by thread {} because of timeout",
+ id,
+ Thread.currentThread().getName());
+ return false;
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.error(
+ "Interrupted while waiting for PipeTaskCoordinator lock, current thread: {}",
+ Thread.currentThread().getName());
+ return false;
+ }
+ }
+
void unlock() {
final Long id = deque.poll();
if (id == null) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
index 892859fd57e..07196c23e1b 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/AbstractOperatePipeProcedureV2.java
@@ -71,45 +71,78 @@ public abstract class AbstractOperatePipeProcedureV2
@Override
protected ProcedureLockState acquireLock(ConfigNodeProcedureEnv configNodeProcedureEnv) {
- configNodeProcedureEnv.getSchedulerLock().lock();
- try {
- if (configNodeProcedureEnv.getNodeLock().tryLock(this)) {
- pipeTaskInfo =
- configNodeProcedureEnv
- .getConfigManager()
- .getPipeManager()
- .getPipeTaskCoordinator()
- .lock();
- LOGGER.info("ProcedureId {} acquire lock.", getProcId());
- return ProcedureLockState.LOCK_ACQUIRED;
- }
- configNodeProcedureEnv.getNodeLock().waitProcedure(this);
- LOGGER.info("ProcedureId {} wait for lock.", getProcId());
- return ProcedureLockState.LOCK_EVENT_WAIT;
- } finally {
- configNodeProcedureEnv.getSchedulerLock().unlock();
+ LOGGER.info("ProcedureId {} try to acquire pipe lock.", getProcId());
+ pipeTaskInfo =
+ configNodeProcedureEnv
+ .getConfigManager()
+ .getPipeManager()
+ .getPipeTaskCoordinator()
+ .tryLock();
+ if (pipeTaskInfo == null) {
+ LOGGER.warn("ProcedureId {} failed to acquire pipe lock.", getProcId());
+ } else {
+ LOGGER.info("ProcedureId {} acquired pipe lock.", getProcId());
+ }
+
+ final ProcedureLockState procedureLockState = super.acquireLock(configNodeProcedureEnv);
+ switch (procedureLockState) {
+ case LOCK_ACQUIRED:
+ if (pipeTaskInfo == null) {
+ LOGGER.warn(
+ "ProcedureId {}: LOCK_ACQUIRED. The following procedure should not be executed without pipe lock.",
+ getProcId());
+ } else {
+ LOGGER.info(
+ "ProcedureId {}: LOCK_ACQUIRED. The following procedure should be executed with pipe lock.",
+ getProcId());
+ }
+ break;
+ case LOCK_EVENT_WAIT:
+ if (pipeTaskInfo == null) {
+ LOGGER.warn("ProcedureId {}: LOCK_EVENT_WAIT. Without acquiring pipe lock.", getProcId());
+ } else {
+ LOGGER.info("ProcedureId {}: LOCK_EVENT_WAIT. Pipe lock will be released.", getProcId());
+ configNodeProcedureEnv
+ .getConfigManager()
+ .getPipeManager()
+ .getPipeTaskCoordinator()
+ .unlock();
+ pipeTaskInfo = null;
+ }
+ break;
+ default:
+ if (pipeTaskInfo == null) {
+ LOGGER.error(
+ "ProcedureId {}: {}. Invalid lock state. Without acquiring pipe lock.",
+ getProcId(),
+ procedureLockState);
+ } else {
+ LOGGER.error(
+ "ProcedureId {}: {}. Invalid lock state. Pipe lock will be released.",
+ getProcId(),
+ procedureLockState);
+ configNodeProcedureEnv
+ .getConfigManager()
+ .getPipeManager()
+ .getPipeTaskCoordinator()
+ .unlock();
+ pipeTaskInfo = null;
+ }
+ break;
}
+ return procedureLockState;
}
@Override
protected void releaseLock(ConfigNodeProcedureEnv configNodeProcedureEnv) {
- configNodeProcedureEnv.getSchedulerLock().lock();
- try {
- LOGGER.info("ProcedureId {} release lock.", getProcId());
- if (pipeTaskInfo != null) {
- configNodeProcedureEnv
- .getConfigManager()
- .getPipeManager()
- .getPipeTaskCoordinator()
- .unlock();
- }
- if (configNodeProcedureEnv.getNodeLock().releaseLock(this)) {
- configNodeProcedureEnv
- .getNodeLock()
- .wakeWaitingProcedures(configNodeProcedureEnv.getScheduler());
- }
- } finally {
- configNodeProcedureEnv.getSchedulerLock().unlock();
+ super.releaseLock(configNodeProcedureEnv);
+
+ if (pipeTaskInfo == null) {
+ LOGGER.warn("ProcedureId {} release lock. No need to release pipe lock.", getProcId());
+ } else {
+ LOGGER.info("ProcedureId {} release lock. Pipe lock will be released.", getProcId());
+ configNodeProcedureEnv.getConfigManager().getPipeManager().getPipeTaskCoordinator().unlock();
+ pipeTaskInfo = null;
}
}
@@ -145,6 +178,13 @@ public abstract class AbstractOperatePipeProcedureV2
@Override
protected Flow executeFromState(ConfigNodeProcedureEnv env, OperatePipeTaskState state)
throws ProcedureSuspendedException, ProcedureYieldException, InterruptedException {
+ if (pipeTaskInfo == null) {
+ LOGGER.warn(
+ "ProcedureId {}: Pipe lock is not acquired, executeFromState's execution will be skipped.",
+ getProcId());
+ return Flow.NO_MORE_STATE;
+ }
+
try {
switch (state) {
case VALIDATE_TASK:
@@ -170,7 +210,8 @@ public abstract class AbstractOperatePipeProcedureV2
// Retry before rollback
if (getCycles() < RETRY_THRESHOLD) {
LOGGER.warn(
- "Encountered error when trying to {} at state [{}], retry [{}/{}]",
+ "ProcedureId {}: Encountered error when trying to {} at state [{}], retry [{}/{}]",
+ getProcId(),
getOperation(),
state,
getCycles() + 1,
@@ -180,14 +221,17 @@ public abstract class AbstractOperatePipeProcedureV2
TimeUnit.MILLISECONDS.sleep(3000L);
} else {
LOGGER.warn(
- "All {} retries failed when trying to {} at state [{}], will rollback...",
+ "ProcedureId {}: All {} retries failed when trying to {} at state [{}], will rollback...",
+ getProcId(),
RETRY_THRESHOLD,
getOperation(),
state,
e);
setFailure(
new ProcedureException(
- String.format("Fail to %s because %s", getOperation().name(), e.getMessage())));
+ String.format(
+ "ProcedureId %s: Fail to %s because %s",
+ getProcId(), getOperation().name(), e.getMessage())));
}
}
return Flow.HAS_MORE_STATE;
@@ -201,29 +245,59 @@ public abstract class AbstractOperatePipeProcedureV2
@Override
protected void rollbackState(ConfigNodeProcedureEnv env, OperatePipeTaskState state)
throws IOException, InterruptedException, ProcedureException {
+ if (pipeTaskInfo == null) {
+ LOGGER.warn(
+ "ProcedureId {}: Pipe lock is not acquired, rollbackState({})'s execution will be skipped.",
+ getProcId(),
+ state);
+ return;
+ }
+
switch (state) {
case VALIDATE_TASK:
- rollbackFromValidateTask(env);
+ try {
+ rollbackFromValidateTask(env);
+ } catch (Exception e) {
+ LOGGER.warn("ProcedureId {}: Failed to rollback from validate task.", getProcId(), e);
+ }
break;
case CALCULATE_INFO_FOR_TASK:
- rollbackFromCalculateInfoForTask(env);
+ try {
+ rollbackFromCalculateInfoForTask(env);
+ } catch (Exception e) {
+ LOGGER.warn(
+ "ProcedureId {}: Failed to rollback from calculate info for task.", getProcId(), e);
+ }
break;
case WRITE_CONFIG_NODE_CONSENSUS:
- // rollbackFromWriteConfigNodeConsensus can be called before rollbackFromOperateOnDataNodes
- // so we need to check if rollbackFromOperateOnDataNodes is successful executed
- // if yes, we don't need to call rollbackFromWriteConfigNodeConsensus again
- if (!isRollbackFromOperateOnDataNodesSuccessful) {
- rollbackFromWriteConfigNodeConsensus(env);
+ try {
+ // rollbackFromWriteConfigNodeConsensus can be called before
+ // rollbackFromOperateOnDataNodes.
+ // So we need to check if rollbackFromOperateOnDataNodes is successfully executed.
+ // If yes, we don't need to call rollbackFromWriteConfigNodeConsensus again.
+ if (!isRollbackFromOperateOnDataNodesSuccessful) {
+ rollbackFromWriteConfigNodeConsensus(env);
+ }
+ } catch (Exception e) {
+ LOGGER.warn(
+ "ProcedureId {}: Failed to rollback from write config node consensus.",
+ getProcId(),
+ e);
}
break;
case OPERATE_ON_DATA_NODES:
- // We have to make sure that rollbackFromOperateOnDataNodes is executed before
- // rollbackFromWriteConfigNodeConsensus, because rollbackFromOperateOnDataNodes is
- // executed based on the consensus of config nodes that is written by
- // rollbackFromWriteConfigNodeConsensus
- rollbackFromWriteConfigNodeConsensus(env);
- rollbackFromOperateOnDataNodes(env);
- isRollbackFromOperateOnDataNodesSuccessful = true;
+ try {
+ // We have to make sure that rollbackFromOperateOnDataNodes is executed before
+ // rollbackFromWriteConfigNodeConsensus, because rollbackFromOperateOnDataNodes is
+ // executed based on the consensus of config nodes that is written by
+ // rollbackFromWriteConfigNodeConsensus
+ rollbackFromWriteConfigNodeConsensus(env);
+ rollbackFromOperateOnDataNodes(env);
+ isRollbackFromOperateOnDataNodesSuccessful = true;
+ } catch (Exception e) {
+ LOGGER.warn(
+ "ProcedureId {}: Failed to rollback from operate on data nodes.", getProcId(), e);
+ }
break;
default:
LOGGER.error("Unsupported roll back STATE [{}]", state);