You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2022/12/16 11:55:13 UTC
[dolphinscheduler] branch dev updated: Solve the deadlock problem caused by queuing (#13191)
This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 7a0a2c2a46 Solve the deadlock problem caused by queuing (#13191)
7a0a2c2a46 is described below
commit 7a0a2c2a46a0224d0c9fb1f649c1f901d11e814b
Author: sssqhai <35...@users.noreply.github.com>
AuthorDate: Fri Dec 16 19:55:02 2022 +0800
Solve the deadlock problem caused by queuing (#13191)
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing
* Solve the deadlock problem caused by queuing,move the event to the tail by throwing a exception
Co-authored-by: wfs <wa...@cdqcp.cpm>
---
...ntHandler.java => StateEventHandleFailure.java} | 25 ++++++++++------------
.../server/master/event/StateEventHandler.java | 3 ++-
.../event/TaskWaitTaskGroupStateHandler.java | 8 +++++--
.../master/runner/WorkflowExecuteRunnable.java | 8 +++++++
4 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
similarity index 53%
copy from dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
copy to dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
index 93d4a7d4b3..5e757c7858 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandleFailure.java
@@ -17,20 +17,17 @@
package org.apache.dolphinscheduler.server.master.event;
-import org.apache.dolphinscheduler.common.enums.StateEventType;
-import org.apache.dolphinscheduler.server.master.runner.WorkflowExecuteRunnable;
-
-public interface StateEventHandler {
+/**
+ * This exception represent the exception can be recovered, when we get this exception,
+ * we will move the event to the fail of the queue.
+ */
+public class StateEventHandleFailure extends Exception {
- /**
- * Handle a event, if handle success will reture true, else return false
- *
- * @param stateEvent given state event.
- * @throws StateEventHandleException this exception means it can be recovered.
- * @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop.
- */
- boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError;
+ public StateEventHandleFailure(String message) {
+ super(message);
+ }
- StateEventType getEventType();
+ public StateEventHandleFailure(String message, Throwable throwable) {
+ super(message, throwable);
+ }
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
index 93d4a7d4b3..377ea71f62 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/StateEventHandler.java
@@ -28,9 +28,10 @@ public interface StateEventHandler {
* @param stateEvent given state event.
* @throws StateEventHandleException this exception means it can be recovered.
* @throws StateEventHandleError this exception means it cannot be recovered, so the event need to drop.
+ * @throws StateEventHandleException this means it can be recovered.
*/
boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
- StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError;
+ StateEvent stateEvent) throws StateEventHandleException, StateEventHandleError, StateEventHandleFailure;
StateEventType getEventType();
}
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
index 02d08a6616..bb0c7b8b70 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/event/TaskWaitTaskGroupStateHandler.java
@@ -31,9 +31,13 @@ public class TaskWaitTaskGroupStateHandler implements StateEventHandler {
private static final Logger logger = LoggerFactory.getLogger(TaskWaitTaskGroupStateHandler.class);
@Override
- public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable, StateEvent stateEvent) {
+ public boolean handleStateEvent(WorkflowExecuteRunnable workflowExecuteRunnable,
+ StateEvent stateEvent) throws StateEventHandleFailure {
logger.info("Handle task instance wait task group event, taskInstanceId: {}", stateEvent.getTaskInstanceId());
- return workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent);
+ if (!workflowExecuteRunnable.checkForceStartAndWakeUp(stateEvent)) {
+ throw new StateEventHandleFailure("Task state event handle failed due to robing taskGroup resource failed");
+ }
+ return true;
}
@Override
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
index efb4a944c7..ab0af2be5b 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/WorkflowExecuteRunnable.java
@@ -71,6 +71,7 @@ import org.apache.dolphinscheduler.server.master.dispatch.executor.NettyExecutor
import org.apache.dolphinscheduler.server.master.event.StateEvent;
import org.apache.dolphinscheduler.server.master.event.StateEventHandleError;
import org.apache.dolphinscheduler.server.master.event.StateEventHandleException;
+import org.apache.dolphinscheduler.server.master.event.StateEventHandleFailure;
import org.apache.dolphinscheduler.server.master.event.StateEventHandler;
import org.apache.dolphinscheduler.server.master.event.StateEventHandlerManager;
import org.apache.dolphinscheduler.server.master.event.TaskStateEvent;
@@ -315,6 +316,13 @@ public class WorkflowExecuteRunnable implements Callable<WorkflowSubmitStatue> {
stateEvent,
stateEventHandleException);
ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
+ } catch (StateEventHandleFailure stateEventHandleFailure) {
+ logger.error("State event handle failed, will move event to the tail: {}",
+ stateEvent,
+ stateEventHandleFailure);
+ this.stateEvents.remove(stateEvent);
+ this.stateEvents.offer(stateEvent);
+ ThreadUtils.sleep(Constants.SLEEP_TIME_MILLIS);
} catch (Exception e) {
// we catch the exception here, since if the state event handle failed, the state event will still keep
// in the stateEvents queue.